From e73e8e42375ce54f07ebeb6e94a71758edc725ad Mon Sep 17 00:00:00 2001 From: abdarrahman abouzaid Date: Wed, 3 Jun 2020 15:19:05 +0200 Subject: [PATCH 1/7] re-disable caches that were disabled before setup:upgrade command --- setup/src/Magento/Setup/Model/Installer.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php index 535040f942b89..efa27c6149060 100644 --- a/setup/src/Magento/Setup/Model/Installer.php +++ b/setup/src/Magento/Setup/Model/Installer.php @@ -875,6 +875,15 @@ public function installDataFixtures(array $request = []) LayoutCache::TYPE_IDENTIFIER, ]; + $cacheManager = $this->objectManagerProvider->get()->create(\Magento\Framework\App\Cache\Manager::class); + $disabledCaches = array_filter( + $cacheManager->getStatus(), + function ($value, string $key) use ($frontendCaches) { + return $value == false && in_array($key, $frontendCaches); + }, + ARRAY_FILTER_USE_BOTH + ); + /** @var \Magento\Framework\Registry $registry */ $registry = $this->objectManagerProvider->get()->get(\Magento\Framework\Registry::class); //For backward compatibility in install and upgrade scripts with enabled parallelization. @@ -888,8 +897,10 @@ public function installDataFixtures(array $request = []) $this->log->log('Disabling caches:'); $this->updateCaches(false, $frontendCaches); $this->handleDBSchemaData($setup, 'data', $request); - $this->log->log('Enabling caches:'); + $this->log->log('Enabling Caches:'); $this->updateCaches(true, $frontendCaches); + $this->log->log('Return Disabled Caches to their old state:'); + $this->updateCaches(false, array_keys($disabledCaches)); $registry->unregister('setup-mode-enabled'); } From ec9462e200021ebc0fd24fce64a15fe51d9062cd Mon Sep 17 00:00:00 2001 From: abdarrahman abouzaid Date: Wed, 3 Jun 2020 15:31:38 +0200 Subject: [PATCH 2/7] lowercase the un needed uppercase words --- setup/src/Magento/Setup/Model/Installer.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php index efa27c6149060..c6a3aba3c07f4 100644 --- a/setup/src/Magento/Setup/Model/Installer.php +++ b/setup/src/Magento/Setup/Model/Installer.php @@ -897,9 +897,9 @@ function ($value, string $key) use ($frontendCaches) { $this->log->log('Disabling caches:'); $this->updateCaches(false, $frontendCaches); $this->handleDBSchemaData($setup, 'data', $request); - $this->log->log('Enabling Caches:'); + $this->log->log('Enabling caches:'); $this->updateCaches(true, $frontendCaches); - $this->log->log('Return Disabled Caches to their old state:'); + $this->log->log('Return disabled caches back to their old state:'); $this->updateCaches(false, array_keys($disabledCaches)); $registry->unregister('setup-mode-enabled'); From d8c0afd344e28c33a58e6416535ed48090e50da4 Mon Sep 17 00:00:00 2001 From: abdarrahman abouzaid Date: Thu, 4 Jun 2020 13:02:28 +0200 Subject: [PATCH 3/7] confirm changes for only upgrade command --- .../Setup/Console/Command/UpgradeCommand.php | 2 +- setup/src/Magento/Setup/Model/Installer.php | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php index 55081c5a9c9a4..8c3ca138e7654 100644 --- a/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php +++ b/setup/src/Magento/Setup/Console/Command/UpgradeCommand.php @@ -120,7 +120,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $installer = $this->installerFactory->create(new ConsoleLogger($output)); $installer->updateModulesSequence($keepGenerated); $installer->installSchema($request); - $installer->installDataFixtures($request); + $installer->installDataFixtures($request, true); if ($this->deploymentConfig->isAvailable()) { $importConfigCommand = $this->getApplication()->find(ConfigImportCommand::COMMAND_NAME); diff --git a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php index c6a3aba3c07f4..b471379b3bcfc 100644 --- a/setup/src/Magento/Setup/Model/Installer.php +++ b/setup/src/Magento/Setup/Model/Installer.php @@ -867,7 +867,7 @@ private function convertationOfOldScriptsIsAllowed(array $request) * @param array $request * @return void */ - public function installDataFixtures(array $request = []) + public function installDataFixtures(array $request = [], $keepCacheStatuses = false) { $frontendCaches = [ PageCache::TYPE_IDENTIFIER, @@ -875,14 +875,16 @@ public function installDataFixtures(array $request = []) LayoutCache::TYPE_IDENTIFIER, ]; - $cacheManager = $this->objectManagerProvider->get()->create(\Magento\Framework\App\Cache\Manager::class); - $disabledCaches = array_filter( - $cacheManager->getStatus(), - function ($value, string $key) use ($frontendCaches) { - return $value == false && in_array($key, $frontendCaches); - }, - ARRAY_FILTER_USE_BOTH - ); + if($keepCacheStatuses){ + $cacheManager = $this->objectManagerProvider->get()->create(\Magento\Framework\App\Cache\Manager::class); + $disabledCaches = array_filter( + $cacheManager->getStatus(), + function ($value, string $key) use ($frontendCaches) { + return $value == false && in_array($key, $frontendCaches); + }, + ARRAY_FILTER_USE_BOTH + ); + } /** @var \Magento\Framework\Registry $registry */ $registry = $this->objectManagerProvider->get()->get(\Magento\Framework\Registry::class); @@ -899,8 +901,10 @@ function ($value, string $key) use ($frontendCaches) { $this->handleDBSchemaData($setup, 'data', $request); $this->log->log('Enabling caches:'); $this->updateCaches(true, $frontendCaches); - $this->log->log('Return disabled caches back to their old state:'); - $this->updateCaches(false, array_keys($disabledCaches)); + if($keepCacheStatuses && !empty($disabledCaches)){ + $this->log->log('Disabling pre-disabled caches:'); + $this->updateCaches(false, array_keys($disabledCaches)); + } $registry->unregister('setup-mode-enabled'); } From 6a82be16a1371116a92ef9dfb5e6dc09fd3bfe51 Mon Sep 17 00:00:00 2001 From: abdarrahman abouzaid Date: Thu, 4 Jun 2020 14:51:12 +0200 Subject: [PATCH 4/7] update code style --- setup/src/Magento/Setup/Model/Installer.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php index b471379b3bcfc..f0ea4061ca9b9 100644 --- a/setup/src/Magento/Setup/Model/Installer.php +++ b/setup/src/Magento/Setup/Model/Installer.php @@ -865,6 +865,7 @@ private function convertationOfOldScriptsIsAllowed(array $request) * Installs data fixtures * * @param array $request + * @param boolean $keepCacheStatuses * @return void */ public function installDataFixtures(array $request = [], $keepCacheStatuses = false) @@ -875,7 +876,7 @@ public function installDataFixtures(array $request = [], $keepCacheStatuses = fa LayoutCache::TYPE_IDENTIFIER, ]; - if($keepCacheStatuses){ + if ($keepCacheStatuses) { $cacheManager = $this->objectManagerProvider->get()->create(\Magento\Framework\App\Cache\Manager::class); $disabledCaches = array_filter( $cacheManager->getStatus(), @@ -901,7 +902,7 @@ function ($value, string $key) use ($frontendCaches) { $this->handleDBSchemaData($setup, 'data', $request); $this->log->log('Enabling caches:'); $this->updateCaches(true, $frontendCaches); - if($keepCacheStatuses && !empty($disabledCaches)){ + if ($keepCacheStatuses && !empty($disabledCaches)) { $this->log->log('Disabling pre-disabled caches:'); $this->updateCaches(false, array_keys($disabledCaches)); } From dceaeae3397021535b88d8f49c040f44a3d01f93 Mon Sep 17 00:00:00 2001 From: Ihor Sviziev Date: Thu, 27 Aug 2020 13:18:23 +0300 Subject: [PATCH 5/7] Fix setup upgrade command re enables caches - Do not touch disabled cache types - Re-enable cache in case if exception happened in handleDBSchemaData --- setup/src/Magento/Setup/Model/Installer.php | 54 +++++++++++++++------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php index daf8a17553752..a4af1f04067a9 100644 --- a/setup/src/Magento/Setup/Model/Installer.php +++ b/setup/src/Magento/Setup/Model/Installer.php @@ -8,6 +8,7 @@ use Magento\Backend\Setup\ConfigOptionsList as BackendConfigOptionsList; use Magento\Framework\App\Cache\Manager; +use Magento\Framework\App\Cache\Manager as CacheManager; use Magento\Framework\App\Cache\Type\Block as BlockCache; use Magento\Framework\App\Cache\Type\Config as ConfigCache; use Magento\Framework\App\Cache\Type\Layout as LayoutCache; @@ -934,14 +935,9 @@ public function installDataFixtures(array $request = [], $keepCacheStatuses = fa ]; if ($keepCacheStatuses) { - $cacheManager = $this->objectManagerProvider->get()->create(\Magento\Framework\App\Cache\Manager::class); - $disabledCaches = array_filter( - $cacheManager->getStatus(), - function ($value, string $key) use ($frontendCaches) { - return $value == false && in_array($key, $frontendCaches); - }, - ARRAY_FILTER_USE_BOTH - ); + $disabledCaches = $this->getDisabledCacheTypes($frontendCaches); + + $frontendCaches = array_diff($frontendCaches, $disabledCaches); } /** @var \Magento\Framework\Registry $registry */ @@ -954,14 +950,19 @@ function ($value, string $key) use ($frontendCaches) { $setup = $this->dataSetupFactory->create(); $this->checkFilePermissionsForDbUpgrade(); $this->log->log('Data install/update:'); - $this->log->log('Disabling caches:'); - $this->updateCaches(false, $frontendCaches); - $this->handleDBSchemaData($setup, 'data', $request); - $this->log->log('Enabling caches:'); - $this->updateCaches(true, $frontendCaches); - if ($keepCacheStatuses && !empty($disabledCaches)) { - $this->log->log('Disabling pre-disabled caches:'); - $this->updateCaches(false, array_keys($disabledCaches)); + + if ($frontendCaches) { + $this->log->log('Disabling caches:'); + $this->updateCaches(false, $frontendCaches); + } + + try { + $this->handleDBSchemaData($setup, 'data', $request); + } finally { + if ($frontendCaches) { + $this->log->log('Enabling caches:'); + $this->updateCaches(true, $frontendCaches); + } } $registry->unregister('setup-mode-enabled'); @@ -1742,4 +1743,25 @@ public function removeUnusedTriggers(): void $this->triggerCleaner->removeTriggers(); $this->cleanCaches(); } + + /** + * @param array $cacheTypesToCheck + * @return array + */ + private function getDisabledCacheTypes(array $cacheTypesToCheck): array + { + $disabledCaches = []; + + /** @var CacheManager $cacheManager */ + $cacheManager = $this->objectManagerProvider->get()->create(CacheManager::class); + $cacheStatus = $cacheManager->getStatus(); + + foreach ($cacheTypesToCheck as $cacheType) { + if (isset($cacheStatus[$cacheType]) && $cacheStatus[$cacheType] === 0) { + $disabledCaches[] = $cacheType; + } + } + + return $disabledCaches; + } } From e3b8b3b9c78d190c3148280c47f0bebebfe62a13 Mon Sep 17 00:00:00 2001 From: Ihor Sviziev Date: Thu, 27 Aug 2020 13:29:28 +0300 Subject: [PATCH 6/7] Fix setup upgrade command re enables caches Fix static test failure --- setup/src/Magento/Setup/Model/Installer.php | 26 ++++++++++----------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/setup/src/Magento/Setup/Model/Installer.php b/setup/src/Magento/Setup/Model/Installer.php index a4af1f04067a9..734df8ba84e78 100644 --- a/setup/src/Magento/Setup/Model/Installer.php +++ b/setup/src/Magento/Setup/Model/Installer.php @@ -1012,7 +1012,7 @@ private function throwExceptionForNotWritablePaths(array $paths) */ private function handleDBSchemaData($setup, $type, array $request) { - if (!($type === 'schema' || $type === 'data')) { + if ($type !== 'schema' && $type !== 'data') { // phpcs:ignore Magento2.Exceptions.DirectThrow throw new Exception("Unsupported operation type $type is requested"); } @@ -1031,17 +1031,13 @@ private function handleDBSchemaData($setup, $type, array $request) 'objectManager' => $this->objectManagerProvider->get() ] ); + + $patchApplierParams = $type === 'schema' ? + ['schemaSetup' => $setup] : + ['moduleDataSetup' => $setup, 'objectManager' => $this->objectManagerProvider->get()]; + /** @var PatchApplier $patchApplier */ - if ($type === 'schema') { - $patchApplier = $this->patchApplierFactory->create(['schemaSetup' => $setup]); - } elseif ($type === 'data') { - $patchApplier = $this->patchApplierFactory->create( - [ - 'moduleDataSetup' => $setup, - 'objectManager' => $this->objectManagerProvider->get() - ] - ); - } + $patchApplier = $this->patchApplierFactory->create($patchApplierParams); foreach ($moduleNames as $moduleName) { if ($this->isDryRun($request)) { @@ -1103,11 +1099,11 @@ private function handleDBSchemaData($setup, $type, array $request) if ($type === 'schema') { $this->log->log('Schema post-updates:'); - $handlerType = 'schema-recurring'; } elseif ($type === 'data') { $this->log->log('Data post-updates:'); - $handlerType = 'data-recurring'; } + $handlerType = $type === 'schema' ? 'schema-recurring' : 'data-recurring'; + foreach ($moduleNames as $moduleName) { if ($this->isDryRun($request)) { $this->log->log("Module '{$moduleName}':"); @@ -1745,6 +1741,8 @@ public function removeUnusedTriggers(): void } /** + * Returns list of disabled cache types + * * @param array $cacheTypesToCheck * @return array */ @@ -1761,7 +1759,7 @@ private function getDisabledCacheTypes(array $cacheTypesToCheck): array $disabledCaches[] = $cacheType; } } - + return $disabledCaches; } } From aaf944ecc411516ec245226dcf53957351b96feb Mon Sep 17 00:00:00 2001 From: "vadim.malesh" Date: Wed, 16 Sep 2020 11:46:48 +0300 Subject: [PATCH 7/7] test coverage --- .../Setup/Test/Unit/Model/InstallerTest.php | 115 ++++++++++++++++-- 1 file changed, 107 insertions(+), 8 deletions(-) diff --git a/setup/src/Magento/Setup/Test/Unit/Model/InstallerTest.php b/setup/src/Magento/Setup/Test/Unit/Model/InstallerTest.php index 48afa684bb9d2..99d8d566323a1 100644 --- a/setup/src/Magento/Setup/Test/Unit/Model/InstallerTest.php +++ b/setup/src/Magento/Setup/Test/Unit/Model/InstallerTest.php @@ -62,7 +62,18 @@ class InstallerTest extends TestCase { /** - * @var \Magento\Setup\Model\Installer + * @var array + */ + private $request = [ + ConfigOptionsListConstants::INPUT_KEY_DB_HOST => '127.0.0.1', + ConfigOptionsListConstants::INPUT_KEY_DB_NAME => 'magento', + ConfigOptionsListConstants::INPUT_KEY_DB_USER => 'magento', + ConfigOptionsListConstants::INPUT_KEY_ENCRYPTION_KEY => 'encryption_key', + ConfigOptionsList::INPUT_KEY_BACKEND_FRONTNAME => 'backend', + ]; + + /** + * @var Installer */ private $object; @@ -426,13 +437,7 @@ public function installDataProvider() { return [ [ - 'request' => [ - ConfigOptionsListConstants::INPUT_KEY_DB_HOST => '127.0.0.1', - ConfigOptionsListConstants::INPUT_KEY_DB_NAME => 'magento', - ConfigOptionsListConstants::INPUT_KEY_DB_USER => 'magento', - ConfigOptionsListConstants::INPUT_KEY_ENCRYPTION_KEY => 'encryption_key', - ConfigOptionsList::INPUT_KEY_BACKEND_FRONTNAME => 'backend', - ], + 'request' => $this->request, 'logMessages' => [ ['Starting Magento installation:'], ['File permissions check...'], @@ -526,6 +531,100 @@ public function installDataProvider() ]; } + /** + * Test for InstallDataFixtures + * + * @dataProvider testInstallDataFixturesDataProvider + * + * @param bool $keepCache + * @param array $expectedToEnableCacheTypes + * @return void + */ + public function testInstallDataFixtures(bool $keepCache, array $expectedToEnableCacheTypes): void + { + $cacheManagerMock = $this->createMock(Manager::class); + //simulate disabled layout cache type + $cacheManagerMock->expects($this->atLeastOnce()) + ->method('getStatus') + ->willReturn(['layout' => 0]); + $cacheManagerMock->expects($this->atLeastOnce()) + ->method('getAvailableTypes') + ->willReturn(['block_html', 'full_page', 'layout' , 'config', 'collections']); + $cacheManagerMock->expects($this->exactly(2)) + ->method('setEnabled') + ->withConsecutive([$expectedToEnableCacheTypes, false], [$expectedToEnableCacheTypes, true]) + ->willReturn([]); + + $this->objectManager->expects($this->atLeastOnce()) + ->method('create') + ->willReturnMap([ + [Manager::class, [], $cacheManagerMock], + [ + PatchApplierFactory::class, + ['objectManager' => $this->objectManager], + $this->patchApplierFactoryMock + ], + ]); + + $registryMock = $this->createMock(Registry::class); + $this->objectManager->expects($this->atLeastOnce()) + ->method('get') + ->with(Registry::class) + ->willReturn($registryMock); + + $this->config->expects($this->atLeastOnce()) + ->method('get') + ->willReturn(true); + + $this->filePermissions->expects($this->atLeastOnce()) + ->method('getMissingWritableDirectoriesForDbUpgrade') + ->willReturn([]); + + $connection = $this->getMockBuilder(AdapterInterface::class) + ->addMethods(['getSchemaListener']) + ->getMockForAbstractClass(); + $connection->expects($this->once()) + ->method('getSchemaListener') + ->willReturn($this->schemaListenerMock); + + $resource = $this->createMock(ResourceConnection::class); + $resource->expects($this->atLeastOnce()) + ->method('getConnection') + ->willReturn($connection); + $this->contextMock->expects($this->once()) + ->method('getResources') + ->willReturn($resource); + + $dataSetup = $this->createMock(DataSetup::class); + $dataSetup->expects($this->once()) + ->method('getConnection') + ->willReturn($connection); + + $this->dataSetupFactory->expects($this->atLeastOnce()) + ->method('create') + ->willReturn($dataSetup); + + $this->object->installDataFixtures($this->request, $keepCache); + } + + /** + * DataProvider for testInstallDataFixtures + * + * @return array + */ + public function testInstallDataFixturesDataProvider(): array + { + return [ + 'keep cache' => [ + true, ['block_html', 'full_page'] + ], + 'do not keep a cache' => [ + false, + ['block_html', 'full_page', 'layout'] + ], + ]; + } + public function testCheckInstallationFilePermissions() { $this->filePermissions