From 0530503033b4a97bb1e3b61ad0afb9e92750f554 Mon Sep 17 00:00:00 2001 From: Aleksandr Parkhomenko Date: Wed, 29 Jul 2020 15:32:44 +0300 Subject: [PATCH] MM-42: Test and fix functionality (#66) - Added functionality that catches shifting of items between pages - Moved styles to SCSS file - Removed unneeded todos - Implemented functionality to schedule sync on changing integration - Implemented functionality to remove non-existed records - Removed unneeded logic --- .../SalesChannelInGroupHandler.php | 3 - .../WebsiteToSalesChannelMappingItemDTO.php | 2 +- .../Magento2Bundle/Entity/ProductTaxClass.php | 2 +- .../Repository/NotInOriginIdsInterface.php | 18 ++ .../Repository/ProductTaxClassRepository.php | 26 +++ .../Entity/Repository/StoreRepository.php | 20 ++- .../Entity/Repository/WebsiteRepository.php | 38 ++++- .../Entity/WebsiteIntegrationStatus.php | 1 + .../Doctrine/Magento2TransportListener.php | 68 ++++++++ .../SalesChannelReverseSyncListener.php | 54 ++++-- .../Form/Type/TransportSettingFormType.php | 6 - .../Type/WebsiteToSalesChannelMappingType.php | 4 - .../Handler/TransportHandler.php | 2 - .../Reader/OriginIdsContextReader.php | 69 ++++++++ .../Remover/NonExistedRecordsRemover.php | 156 ++++++++++++++++++ .../Remover/WebsiteMappingRecordsRemover.php | 98 +++++++++++ .../DefaultMagento2ImportStrategy.php | 74 ++++++++- .../Model/Magento2TransportSettings.php | 20 ++- .../Provider/WebsitesProvider.php | 12 -- .../Resources/config/batch_jobs.yml | 31 +++- .../Resources/config/importexport.yml | 33 +++- .../Resources/config/oro/assets.yml | 3 + .../Resources/config/services.yml | 23 ++- .../Resources/public/css/scss/main.scss | 3 + .../Resources/views/Form/fields.html.twig | 3 +- .../Transport/Rest/Client/SearchClient.php | 20 +-- .../Rest/Client/SearchClientFactory.php | 5 +- .../Rest/Iterator/AbstractSearchIterator.php | 18 +- ...bstractSearchWithShiftCheckingIterator.php | 134 +++++++++++++++ .../Transport/Rest/Iterator/OrderIterator.php | 23 ++- .../Transport/Rest/Request/SearchRequest.php | 5 + .../ShiftedItemsSearchRequestFactory.php | 34 ++++ ...ftedItemsSearchRequestFactoryInterface.php | 16 ++ .../FilterSearchValueConverter.php | 23 --- .../Rest/SearchCriteria/SearchCriteria.php | 16 ++ .../Transport/RestTransport.php | 18 +- 36 files changed, 963 insertions(+), 118 deletions(-) create mode 100644 src/Marello/Bundle/Magento2Bundle/Entity/Repository/NotInOriginIdsInterface.php create mode 100644 src/Marello/Bundle/Magento2Bundle/Entity/Repository/ProductTaxClassRepository.php create mode 100644 src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/Magento2TransportListener.php create mode 100644 src/Marello/Bundle/Magento2Bundle/ImportExport/Reader/OriginIdsContextReader.php create mode 100644 src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/NonExistedRecordsRemover.php create mode 100644 src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/WebsiteMappingRecordsRemover.php create mode 100644 src/Marello/Bundle/Magento2Bundle/Resources/config/oro/assets.yml create mode 100644 src/Marello/Bundle/Magento2Bundle/Resources/public/css/scss/main.scss create mode 100644 src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchWithShiftCheckingIterator.php create mode 100644 src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactory.php create mode 100644 src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactoryInterface.php delete mode 100644 src/Marello/Bundle/Magento2Bundle/Transport/Rest/SearchCriteria/FilterSearchValueConverter.php diff --git a/src/Marello/Bundle/Magento2Bundle/Autocomplete/SalesChannelInGroupHandler.php b/src/Marello/Bundle/Magento2Bundle/Autocomplete/SalesChannelInGroupHandler.php index 39e904f88..abde8a6c7 100644 --- a/src/Marello/Bundle/Magento2Bundle/Autocomplete/SalesChannelInGroupHandler.php +++ b/src/Marello/Bundle/Magento2Bundle/Autocomplete/SalesChannelInGroupHandler.php @@ -6,9 +6,6 @@ use Marello\Bundle\SalesBundle\Entity\Repository\SalesChannelRepository; use Oro\Bundle\FormBundle\Autocomplete\SearchHandler; -/** - * @todo Cover with functional test - */ class SalesChannelInGroupHandler extends SearchHandler { private const DELIMITER = ';'; diff --git a/src/Marello/Bundle/Magento2Bundle/DTO/WebsiteToSalesChannelMappingItemDTO.php b/src/Marello/Bundle/Magento2Bundle/DTO/WebsiteToSalesChannelMappingItemDTO.php index fafb474e6..d3a04f1c8 100644 --- a/src/Marello/Bundle/Magento2Bundle/DTO/WebsiteToSalesChannelMappingItemDTO.php +++ b/src/Marello/Bundle/Magento2Bundle/DTO/WebsiteToSalesChannelMappingItemDTO.php @@ -46,7 +46,7 @@ public function __construct(array $data) /** * @return int */ - public function getOriginWebsiteId(): int + public function getWebsiteOriginId(): int { return $this->data['originWebsiteId']; } diff --git a/src/Marello/Bundle/Magento2Bundle/Entity/ProductTaxClass.php b/src/Marello/Bundle/Magento2Bundle/Entity/ProductTaxClass.php index d4f2642c7..00ad6f53a 100644 --- a/src/Marello/Bundle/Magento2Bundle/Entity/ProductTaxClass.php +++ b/src/Marello/Bundle/Magento2Bundle/Entity/ProductTaxClass.php @@ -7,7 +7,7 @@ use Oro\Bundle\EntityConfigBundle\Metadata\Annotation\Config; /** - * @ORM\Entity + * @ORM\Entity(repositoryClass="Marello\Bundle\Magento2Bundle\Entity\Repository\ProductTaxClassRepository") * @ORM\Table(name="marello_m2_product_tax_class") * @Config() */ diff --git a/src/Marello/Bundle/Magento2Bundle/Entity/Repository/NotInOriginIdsInterface.php b/src/Marello/Bundle/Magento2Bundle/Entity/Repository/NotInOriginIdsInterface.php new file mode 100644 index 000000000..77c344970 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/Entity/Repository/NotInOriginIdsInterface.php @@ -0,0 +1,18 @@ +createQueryBuilder('m2pt'); + $qb + ->select('m2pt') + ->where($qb->expr()->notIn('m2pt.originId', ':existedRecordsOriginIds')) + ->andWhere($qb->expr()->eq('m2pt.channel', ':integrationId')) + ->setParameter('existedRecordsOriginIds', $existedRecordsOriginIds) + ->setParameter('integrationId', $integrationId); + + return $qb->getQuery()->getResult(); + } +} diff --git a/src/Marello/Bundle/Magento2Bundle/Entity/Repository/StoreRepository.php b/src/Marello/Bundle/Magento2Bundle/Entity/Repository/StoreRepository.php index b2612c14c..d690483bb 100644 --- a/src/Marello/Bundle/Magento2Bundle/Entity/Repository/StoreRepository.php +++ b/src/Marello/Bundle/Magento2Bundle/Entity/Repository/StoreRepository.php @@ -4,7 +4,7 @@ use Doctrine\ORM\EntityRepository; -class StoreRepository extends EntityRepository +class StoreRepository extends EntityRepository implements NotInOriginIdsInterface { /** * @param int $websiteId @@ -22,4 +22,22 @@ public function getOriginStoreIdsByWebsiteId(int $websiteId): array return \array_column($result, 'originId'); } + + /** + * {@inheritDoc} + */ + public function getEntitiesNotInOriginIdsInGivenIntegration( + array $existedRecordsOriginIds, + int $integrationId + ): array { + $qb = $this->createQueryBuilder('m2s'); + $qb + ->select('m2s') + ->where($qb->expr()->notIn('m2s.originId', ':existedRecordsOriginIds')) + ->andWhere($qb->expr()->eq('m2s.channel', ':integrationId')) + ->setParameter('existedRecordsOriginIds', $existedRecordsOriginIds) + ->setParameter('integrationId', $integrationId); + + return $qb->getQuery()->getResult(); + } } diff --git a/src/Marello/Bundle/Magento2Bundle/Entity/Repository/WebsiteRepository.php b/src/Marello/Bundle/Magento2Bundle/Entity/Repository/WebsiteRepository.php index f2007737b..f10339501 100644 --- a/src/Marello/Bundle/Magento2Bundle/Entity/Repository/WebsiteRepository.php +++ b/src/Marello/Bundle/Magento2Bundle/Entity/Repository/WebsiteRepository.php @@ -5,7 +5,7 @@ use Doctrine\ORM\EntityRepository; use Marello\Bundle\Magento2Bundle\Model\SalesChannelInfo; -class WebsiteRepository extends EntityRepository +class WebsiteRepository extends EntityRepository implements NotInOriginIdsInterface { /** * @return SalesChannelInfo[] @@ -61,4 +61,40 @@ public function getWebsitesIdsByIntegrationId(int $integrationId): array return \array_column($result, 'id'); } + + /** + * {@inheritDoc} + */ + public function getEntitiesNotInOriginIdsInGivenIntegration( + array $existedRecordsOriginIds, + int $integrationId + ): array { + $qb = $this->createQueryBuilder('m2w'); + $qb + ->select('m2w') + ->where($qb->expr()->notIn('m2w.originId', ':existedRecordsOriginIds')) + ->andWhere($qb->expr()->eq('m2w.channel', ':integrationId')) + ->setParameter('existedRecordsOriginIds', $existedRecordsOriginIds) + ->setParameter('integrationId', $integrationId) + ; + + return $qb->getQuery()->getResult(); + } + + /** + * @param int $integrationId + * @return array + */ + public function getOriginIdsByIntegrationId(int $integrationId): array + { + $qb = $this->createQueryBuilder('m2w'); + $qb + ->select('m2w.originId') + ->where($qb->expr()->eq('m2w.channel', ':integrationId')) + ->setParameter('integrationId', $integrationId); + + $result = $qb->getQuery()->getArrayResult(); + + return \array_column($result, 'originId'); + } } diff --git a/src/Marello/Bundle/Magento2Bundle/Entity/WebsiteIntegrationStatus.php b/src/Marello/Bundle/Magento2Bundle/Entity/WebsiteIntegrationStatus.php index be18e03c4..4586876f6 100644 --- a/src/Marello/Bundle/Magento2Bundle/Entity/WebsiteIntegrationStatus.php +++ b/src/Marello/Bundle/Magento2Bundle/Entity/WebsiteIntegrationStatus.php @@ -35,6 +35,7 @@ class WebsiteIntegrationStatus /** * @todo Rename this to innerStatus + * @todo Update repository after that * * @var Status * diff --git a/src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/Magento2TransportListener.php b/src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/Magento2TransportListener.php new file mode 100644 index 000000000..d18f812ec --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/Magento2TransportListener.php @@ -0,0 +1,68 @@ +genuineSyncScheduler = $genuineSyncScheduler; + $this->trackedTransportPropertyNames = $trackedTransportPropertyNames; + } + + /** + * @param Magento2Transport $transport + * @param LifecycleEventArgs $args + */ + public function postUpdate(Magento2Transport $transport, LifecycleEventArgs $args) + { + if (false === $transport->getChannel()->isEnabled()) { + return; + } + + $changeSet = $args->getEntityManager()->getUnitOfWork()->getEntityChangeSet($transport); + $isTrackablePropertiesyChanged = [] !== \array_intersect( + $this->trackedTransportPropertyNames, + \array_keys($changeSet) + ); + + if (!$isTrackablePropertiesyChanged) { + return; + } + + $this->integrationIdsOnSync[$transport->getChannel()->getId()] = $transport->getChannel()->getId(); + } + + public function postFlush() + { + foreach ($this->integrationIdsOnSync as $integrationId) { + $this->genuineSyncScheduler->schedule($integrationId); + } + + $this->integrationIdsOnSync = []; + } + + public function onClear() + { + $this->integrationIdsOnSync = []; + } +} diff --git a/src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/SalesChannelReverseSyncListener.php b/src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/SalesChannelReverseSyncListener.php index dc4e3f796..e2c1a2543 100644 --- a/src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/SalesChannelReverseSyncListener.php +++ b/src/Marello/Bundle/Magento2Bundle/EventListener/Doctrine/SalesChannelReverseSyncListener.php @@ -13,6 +13,7 @@ use Marello\Bundle\Magento2Bundle\Stack\ProductChangesByChannelStack; use Marello\Bundle\ProductBundle\Entity\Repository\ProductRepository; use Marello\Bundle\SalesBundle\Entity\SalesChannel; +use Oro\Bundle\IntegrationBundle\Manager\GenuineSyncScheduler; use Oro\Component\MessageQueue\Client\MessageProducerInterface; use Oro\Component\MessageQueue\Transport\Exception\Exception; @@ -32,6 +33,9 @@ class SalesChannelReverseSyncListener /** @var MessageProducerInterface */ protected $producer; + /** @var GenuineSyncScheduler */ + protected $genuineSyncScheduler; + /** @var array */ protected $integrationChannelIdsWithProductIds = []; @@ -43,17 +47,20 @@ class SalesChannelReverseSyncListener * @param ProductChangesByChannelStack $changesByChannelStack * @param ProductRepository $productRepository * @param MessageProducerInterface $producer + * @param GenuineSyncScheduler $genuineSyncScheduler */ public function __construct( TrackedSalesChannelProvider $salesChannelInfosProvider, ProductChangesByChannelStack $changesByChannelStack, ProductRepository $productRepository, - MessageProducerInterface $producer + MessageProducerInterface $producer, + GenuineSyncScheduler $genuineSyncScheduler ) { $this->salesChannelProvider = $salesChannelInfosProvider; $this->changesByChannelStack = $changesByChannelStack; $this->productRepository = $productRepository; $this->producer = $producer; + $this->genuineSyncScheduler = $genuineSyncScheduler; } /** @@ -112,6 +119,23 @@ protected function loadProductIdsOfRemovedSalesChannels(UnitOfWork $unitOfWork) public function postFlush(PostFlushEventArgs $args) { $this->salesChannelProvider->clearCache(); + $this->processIntegrationChannelIdsWithProductIds(); + $this->processSalesChannelWithUpdatedActiveField(); + } + + /** + * Clear object storage when error was occurred during UOW#Commit + * + * @param OnClearEventArgs $args + */ + public function onClear(OnClearEventArgs $args) + { + $this->salesChannelsWithUpdatedActiveField = []; + $this->integrationChannelIdsWithProductIds = []; + } + + protected function processIntegrationChannelIdsWithProductIds(): void + { foreach ($this->integrationChannelIdsWithProductIds as $integrationId => $modifiedProductIds) { if (empty($modifiedProductIds)) { continue; @@ -142,11 +166,11 @@ public function postFlush(PostFlushEventArgs $args) } $this->integrationChannelIdsWithProductIds = []; + } - /** - * @todo Extend this logic to call initial sync on integration - * that has enabled sales channels - */ + protected function processSalesChannelWithUpdatedActiveField(): void + { + $integrationIdsOnSync = []; foreach ($this->salesChannelsWithUpdatedActiveField as $salesChannel) { $integrationId = $this->salesChannelProvider->getIntegrationIdBySalesChannelId( $salesChannel->getId(), @@ -160,6 +184,13 @@ public function postFlush(PostFlushEventArgs $args) continue; } + /** + * In case when new sales channel enabled, we should send message on sync + */ + if ($salesChannel->getActive()) { + $integrationIdsOnSync[] = $integrationId; + } + $modifiedProductIds = $this->productRepository->getProductIdsBySalesChannelIds([$salesChannel->getId()]); if (empty($modifiedProductIds)) { continue; @@ -198,18 +229,11 @@ public function postFlush(PostFlushEventArgs $args) ); } - $this->salesChannelsWithUpdatedActiveField = []; - } + foreach ($integrationIdsOnSync as $integrationId) { + $this->genuineSyncScheduler->schedule($integrationId); + } - /** - * Clear object storage when error was occurred during UOW#Commit - * - * @param OnClearEventArgs $args - */ - public function onClear(OnClearEventArgs $args) - { $this->salesChannelsWithUpdatedActiveField = []; - $this->integrationChannelIdsWithProductIds = []; } /** diff --git a/src/Marello/Bundle/Magento2Bundle/Form/Type/TransportSettingFormType.php b/src/Marello/Bundle/Magento2Bundle/Form/Type/TransportSettingFormType.php index a624306d9..da2af9e6a 100644 --- a/src/Marello/Bundle/Magento2Bundle/Form/Type/TransportSettingFormType.php +++ b/src/Marello/Bundle/Magento2Bundle/Form/Type/TransportSettingFormType.php @@ -65,9 +65,6 @@ public function buildForm(FormBuilderInterface $builder, array $options) ] ); - /** - * @todo Schedule initial sync on changing this value - */ $builder->add( 'initialSyncStartDate', OroDateType::class, @@ -128,9 +125,6 @@ public function buildForm(FormBuilderInterface $builder, array $options) ] ); - /** - * @todo Schedule initial sync on changing this value - */ $builder->add( 'websiteToSalesChannelMapping', WebsiteToSalesChannelMappingType::class, diff --git a/src/Marello/Bundle/Magento2Bundle/Form/Type/WebsiteToSalesChannelMappingType.php b/src/Marello/Bundle/Magento2Bundle/Form/Type/WebsiteToSalesChannelMappingType.php index b1797f8a7..a13e3b23b 100644 --- a/src/Marello/Bundle/Magento2Bundle/Form/Type/WebsiteToSalesChannelMappingType.php +++ b/src/Marello/Bundle/Magento2Bundle/Form/Type/WebsiteToSalesChannelMappingType.php @@ -25,10 +25,6 @@ public function getBlockPrefix() */ public function buildForm(FormBuilderInterface $builder, array $options) { - /** - * @todo Implement transforming data to DTO for intermidiate validation - */ -// $builder->addModelTransformer(new ArrayToJsonTransformer()); $builder->addViewTransformer(new ArrayToJsonTransformer()); } diff --git a/src/Marello/Bundle/Magento2Bundle/Handler/TransportHandler.php b/src/Marello/Bundle/Magento2Bundle/Handler/TransportHandler.php index a23f5371c..6c1602f5e 100644 --- a/src/Marello/Bundle/Magento2Bundle/Handler/TransportHandler.php +++ b/src/Marello/Bundle/Magento2Bundle/Handler/TransportHandler.php @@ -84,8 +84,6 @@ public function getCheckResponse( } /** - * @todo Replace with validator - * * @param Magento2Transport $transportEntity * @return bool */ diff --git a/src/Marello/Bundle/Magento2Bundle/ImportExport/Reader/OriginIdsContextReader.php b/src/Marello/Bundle/Magento2Bundle/ImportExport/Reader/OriginIdsContextReader.php new file mode 100644 index 000000000..39c409a63 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/ImportExport/Reader/OriginIdsContextReader.php @@ -0,0 +1,69 @@ +innerIterator) { + $this->logger->notice( + '[Magento 2] The OriginIdsContextReader hasn\'t configured properly. '. + 'Expected innerIterator set, but got null.' + ); + + return null; + } + + $result = null; + if ($this->innerIterator->valid()) { + $result = $this->innerIterator->current(); + $context = $this->getContext(); + $context->incrementReadOffset(); + $context->incrementReadCount(); + $this->innerIterator->next(); + } + + return $result; + } + + /** + * {@inheritDoc} + */ + protected function initializeFromContext(ContextInterface $context) + { + $originIds = $this->getDataByContextKey( + DefaultMagento2ImportStrategy::CONTEXT_ORIGIN_IDS_OF_IMPORTED_RECORDS + ); + + if (!empty($originIds) && \is_array($originIds)) { + $this->innerIterator = new \ArrayIterator([$originIds]); + } else { + $this->innerIterator = new \ArrayIterator([]); + } + } + + /** + * @param string $contextKey + * @return mixed|null + */ + protected function getDataByContextKey(string $contextKey) + { + $executionContext = $this->getStepExecution()->getJobExecution()->getExecutionContext(); + return $executionContext->get($contextKey); + } +} diff --git a/src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/NonExistedRecordsRemover.php b/src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/NonExistedRecordsRemover.php new file mode 100644 index 000000000..95ffb9a36 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/NonExistedRecordsRemover.php @@ -0,0 +1,156 @@ +registry = $registry; + $this->contextRegistry = $contextRegistry; + } + + /** + * {@inheritDoc} + */ + public function write(array $items) + { + $channelId = $this->getChannelId(); + $existedOriginIds = reset($items); + if(empty($existedOriginIds) || null === $channelId) { + return; + } + + $entityManager = $this->getEntityManager(); + if (null === $entityManager) { + return; + } + + $repository = $this->getRepository($entityManager); + if (null === $repository) { + return; + } + + try { + $entities = $repository->getEntitiesNotInOriginIdsInGivenIntegration($existedOriginIds, $channelId); + if (!empty($entities)) { + foreach ($entities as $index => $entity) { + $entityManager->remove($entity); + if ($index + 1 % 100 === 0) { + $entityManager->flush(); + } + } + + $entityManager->flush(); + $entityManager->clear(); + } + } catch (RetryableException $e) { + $context = $this->contextRegistry->getByStepExecution($this->stepExecution); + $context->setValue('deadlockDetected', true); + } + } + + /** + * @param StepExecution $stepExecution + */ + public function setStepExecution(StepExecution $stepExecution) + { + $this->stepExecution = $stepExecution; + } + + /** + * @return ObjectManager|null + */ + protected function getEntityManager(): ?ObjectManager + { + $className = $this->getContext()->getOption('entityName'); + if (null === $className) { + $this->logger->notice( + '[Magento 2] Trying to call NonExistedRecordsRemover with non properly configured context. ' . + 'Expected to have "entityName" in context, but got null.' + ); + + return null; + } + + return $this->registry->getManagerForClass($className); + } + + /** + * @param ObjectManager $manager + * @return NotInOriginIdsInterface|null + */ + protected function getRepository(ObjectManager $manager): ?NotInOriginIdsInterface + { + $className = $this->getContext()->getOption('entityName'); + if (null === $className) { + $this->logger->notice( + '[Magento 2] Trying to call NonExistedRecordsRemover with non properly configured context. ' . + 'Expected to have "entityName" in context, but got null.' + ); + + return null; + } + + $repository = $manager->getRepository($className); + + if (!$repository instanceof NotInOriginIdsInterface) { + $this->logger->notice( + '[Magento 2] Trying to call NonExistedRecordsRemover with non properly configured context. ' . + 'Expected that entity has repository instanceof RemoveNonExistedRecordsNotInOriginIdsInterface,' . + 'but given doesn\'t implemented it.', + [ + 'entityName' => $className, + 'repositoryType' => is_object($repository) ? get_class($repository) : gettype($repository) + ] + ); + + return null; + } + + return $repository; + } + + /** + * @return int|null + */ + protected function getChannelId(): ?int + { + return $this->getContext()->getOption('channel'); + } + + /** + * @return ContextInterface + */ + protected function getContext() + { + return $this->contextRegistry->getByStepExecution($this->stepExecution); + } +} diff --git a/src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/WebsiteMappingRecordsRemover.php b/src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/WebsiteMappingRecordsRemover.php new file mode 100644 index 000000000..6cc3279d3 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/ImportExport/Remover/WebsiteMappingRecordsRemover.php @@ -0,0 +1,98 @@ +getTransport(); + if (null === $transport) { + return; + } + + $websiteOriginIds = $this->getWebsiteOriginIds(); + /** + * Skip empty list of $websiteOriginIds, because it's probably issue in sync + */ + if (empty($websiteOriginIds)) { + return; + } + + $this->processSavingNewWebsiteToSchMapping($transport, $websiteOriginIds); + } + + /** + * @param Magento2Transport $transport + * @param array $websiteOriginIds + */ + protected function processSavingNewWebsiteToSchMapping(Magento2Transport $transport, array $websiteOriginIds) + { + $newWebsiteToSchMapping = $transport->getSettingsBag()->getMappingItemArrayContainWebsiteOriginId( + $websiteOriginIds + ); + + if (count($transport->getWebsiteToSalesChannelMapping()) === count($newWebsiteToSchMapping)) { + return; + } + + $transport->setWebsiteToSalesChannelMapping($newWebsiteToSchMapping); + $em = $this->registry->getManagerForClass(Magento2Transport::class); + $em->flush(); + $em->clear(); + } + + /** + * @return Magento2Transport|null + */ + protected function getTransport(): ?Magento2Transport + { + $channelId = $this->getChannelId(); + if (null === $channelId) { + return null; + } + + $channel = $this->registry + ->getManagerForClass(Channel::class) + ->find(Channel::class, $channelId); + + if (null === $channel) { + return null; + } + + $transport = $channel->getTransport(); + if ($transport instanceof Magento2Transport) { + return $transport; + } + + return null; + } + + /** + * @return array + */ + protected function getWebsiteOriginIds(): array + { + $channelId = $this->getChannelId(); + if (null === $channelId) { + return []; + } + + /** @var WebsiteRepository $websiteRepository */ + $websiteRepository = $this->registry + ->getManagerForClass(Website::class) + ->getRepository(Website::class); + + return $websiteRepository->getOriginIdsByIntegrationId($channelId); + } +} diff --git a/src/Marello/Bundle/Magento2Bundle/ImportExport/Strategy/DefaultMagento2ImportStrategy.php b/src/Marello/Bundle/Magento2Bundle/ImportExport/Strategy/DefaultMagento2ImportStrategy.php index e2d64adbc..63c3213a8 100644 --- a/src/Marello/Bundle/Magento2Bundle/ImportExport/Strategy/DefaultMagento2ImportStrategy.php +++ b/src/Marello/Bundle/Magento2Bundle/ImportExport/Strategy/DefaultMagento2ImportStrategy.php @@ -2,19 +2,45 @@ namespace Marello\Bundle\Magento2Bundle\ImportExport\Strategy; +use Akeneo\Bundle\BatchBundle\Entity\StepExecution; +use Akeneo\Bundle\BatchBundle\Item\ExecutionContext; +use Akeneo\Bundle\BatchBundle\Step\StepExecutionAwareInterface; use Marello\Bundle\Magento2Bundle\Entity\IntegrationAwareInterface; +use Marello\Bundle\Magento2Bundle\Entity\OriginAwareInterface; use Oro\Bundle\ImportExportBundle\Strategy\Import\ConfigurableAddOrReplaceStrategy; use Psr\Log\LoggerAwareInterface; use Psr\Log\LoggerAwareTrait; use Symfony\Component\PropertyAccess\PropertyAccessor; -class DefaultMagento2ImportStrategy extends ConfigurableAddOrReplaceStrategy implements LoggerAwareInterface +class DefaultMagento2ImportStrategy extends ConfigurableAddOrReplaceStrategy implements + LoggerAwareInterface, + StepExecutionAwareInterface { use LoggerAwareTrait; + /** + * Uses to get info about all existed records of specific type, + * to remove records that non exist anymore in remote system. + */ + public const CONTEXT_ORIGIN_IDS_OF_IMPORTED_RECORDS = 'originIdsOfImportedRecords'; + + /** @var bool */ + protected $enableCollectingOriginIds = false; + /** @var PropertyAccessor */ protected $propertyAccessor; + /** @var StepExecution */ + protected $stepExecution; + + /** + * @param bool $enableCollectingOriginIds + */ + public function setEnableCollectingOriginIds(bool $enableCollectingOriginIds) + { + $this->enableCollectingOriginIds = $enableCollectingOriginIds; + } + /** * @param PropertyAccessor $propertyAccessor */ @@ -23,6 +49,29 @@ public function setPropertyAccessor(PropertyAccessor $propertyAccessor) $this->propertyAccessor = $propertyAccessor; } + /** + * @param StepExecution $stepExecution + */ + public function setStepExecution(StepExecution $stepExecution) + { + $this->stepExecution = $stepExecution; + } + + /** + * {@inheritDoc} + */ + public function process($entity) + { + if ($this->enableCollectingOriginIds && $entity instanceof OriginAwareInterface && $entity->getOriginId()) { + $this->appendDataToContext( + self::CONTEXT_ORIGIN_IDS_OF_IMPORTED_RECORDS, + $entity->getOriginId() + ); + } + + return parent::process($entity); + } + /** * Specify channel as identity field * @@ -56,4 +105,27 @@ protected function combineIdentityValues($entity, $entityClass, array $searchCon return parent::combineIdentityValues($entity, $entityClass, $searchContext); } + + /** + * @return ExecutionContext + */ + protected function getExecutionContext() + { + if (!$this->stepExecution) { + throw new \InvalidArgumentException('Execution context is not configured'); + } + + return $this->stepExecution->getJobExecution()->getExecutionContext(); + } + + /** + * @param string $contextKey + * @param mixed $dataToAppend + */ + protected function appendDataToContext(string $contextKey, $dataToAppend) + { + $data = (array) $this->getExecutionContext()->get($contextKey); + $data[] = $dataToAppend; + $this->getExecutionContext()->put($contextKey, $data); + } } diff --git a/src/Marello/Bundle/Magento2Bundle/Model/Magento2TransportSettings.php b/src/Marello/Bundle/Magento2Bundle/Model/Magento2TransportSettings.php index 2ad1c5133..5d21895ca 100644 --- a/src/Marello/Bundle/Magento2Bundle/Model/Magento2TransportSettings.php +++ b/src/Marello/Bundle/Magento2Bundle/Model/Magento2TransportSettings.php @@ -84,11 +84,29 @@ public function getSalesChannelIdByWebsiteId(int $websiteId): ?int $websiteToSalesChannelMapping = $this->get(self::WEBSITE_TO_SALES_CHANNEL_MAPPING_KEY, []); foreach ($websiteToSalesChannelMapping as $websiteToSalesChannelMappingItem) { $mappingItem = new WebsiteToSalesChannelMappingItemDTO($websiteToSalesChannelMappingItem); - if ($mappingItem->getOriginWebsiteId() === $websiteId) { + if ($mappingItem->getWebsiteOriginId() === $websiteId) { return $mappingItem->getSalesChannelId(); } } return null; } + + /** + * @param int[] $websiteOriginIds + * @return array + */ + public function getMappingItemArrayContainWebsiteOriginId(array $websiteOriginIds): array + { + $filteredMappingItemArray = []; + $websiteToSalesChannelMapping = $this->get(self::WEBSITE_TO_SALES_CHANNEL_MAPPING_KEY, []); + foreach ($websiteToSalesChannelMapping as $websiteToSalesChannelMappingItem) { + $mappingItem = new WebsiteToSalesChannelMappingItemDTO($websiteToSalesChannelMappingItem); + if (\in_array($mappingItem->getWebsiteOriginId(), $websiteOriginIds, true)) { + $filteredMappingItemArray[] = $mappingItem->getData(); + } + } + + return $filteredMappingItemArray; + } } diff --git a/src/Marello/Bundle/Magento2Bundle/Provider/WebsitesProvider.php b/src/Marello/Bundle/Magento2Bundle/Provider/WebsitesProvider.php index fb414bca6..bc4cfad0b 100644 --- a/src/Marello/Bundle/Magento2Bundle/Provider/WebsitesProvider.php +++ b/src/Marello/Bundle/Magento2Bundle/Provider/WebsitesProvider.php @@ -4,21 +4,9 @@ use Marello\Bundle\Magento2Bundle\ImportExport\Converter\WebsiteDataConverter; use Marello\Bundle\Magento2Bundle\Transport\Magento2TransportInterface; -use Symfony\Contracts\Translation\TranslatorInterface; class WebsitesProvider { - /** @var TranslatorInterface */ - protected $translator; - - /** - * @param TranslatorInterface $translator - */ - public function __construct(TranslatorInterface $translator) - { - $this->translator = $translator; - } - /** * @param Magento2TransportInterface $transport * @return array diff --git a/src/Marello/Bundle/Magento2Bundle/Resources/config/batch_jobs.yml b/src/Marello/Bundle/Magento2Bundle/Resources/config/batch_jobs.yml index c14b3c500..963f10e4d 100644 --- a/src/Marello/Bundle/Magento2Bundle/Resources/config/batch_jobs.yml +++ b/src/Marello/Bundle/Magento2Bundle/Resources/config/batch_jobs.yml @@ -14,8 +14,15 @@ connector: writer: oro_importexport.writer.entity parameters: batch_size: 25 - # @todo Implement removing of abandoned website - # @todo Implement updating config WEBSITE_TO_SALES_CHANNEL_MAPPING_KEY + remove_non_existent_data: + title: import + class: Oro\Bundle\BatchBundle\Step\ItemStep + services: + reader: marello_magento2.importexport.reader.origin_ids_context + processor: marello_magento2.importexport.processor.origin_ids_context + writer: marello_magento2.importexport.remover.website_mapping_records + parameters: + batch_size: 1 marello_magento2_store_rest_import: title: "Store import from Magento 2" @@ -30,7 +37,15 @@ connector: writer: oro_importexport.writer.entity parameters: batch_size: 100 - # @todo Implement removing of abandoned stores + remove_non_existent_data: + title: import + class: Oro\Bundle\BatchBundle\Step\ItemStep + services: + reader: marello_magento2.importexport.reader.origin_ids_context + processor: marello_magento2.importexport.processor.origin_ids_context + writer: marello_magento2.importexport.remover.non_existed_records + parameters: + batch_size: 1 marello_magento2_product_tax_class_rest_import: title: "Product Tax Class import from Magento 2" @@ -45,7 +60,15 @@ connector: writer: oro_importexport.writer.entity parameters: batch_size: 25 - # @todo Implement removing of abandoned tax class + remove_non_existent_data: + title: import + class: Oro\Bundle\BatchBundle\Step\ItemStep + services: + reader: marello_magento2.importexport.reader.origin_ids_context + processor: marello_magento2.importexport.processor.origin_ids_context + writer: marello_magento2.importexport.remover.non_existed_records + parameters: + batch_size: 1 marello_magento2_attributeset_rest_import: title: "Attributeset import from Magento 2" diff --git a/src/Marello/Bundle/Magento2Bundle/Resources/config/importexport.yml b/src/Marello/Bundle/Magento2Bundle/Resources/config/importexport.yml index 46a71c62e..8e23f696b 100644 --- a/src/Marello/Bundle/Magento2Bundle/Resources/config/importexport.yml +++ b/src/Marello/Bundle/Magento2Bundle/Resources/config/importexport.yml @@ -10,13 +10,19 @@ services: marello_magento2.import_strategy.website: class: Marello\Bundle\Magento2Bundle\ImportExport\Strategy\WebsiteMagento2ImportStrategy parent: marello_magento2.import_strategy.default_magento2_import + calls: + - ['setEnableCollectingOriginIds', [true]] marello_magento2.import_strategy.store: class: Marello\Bundle\Magento2Bundle\ImportExport\Strategy\StoreMagento2ImportStrategy parent: marello_magento2.import_strategy.default_magento2_import + calls: + - ['setEnableCollectingOriginIds', [true]] marello_magento2.import_strategy.product_tax_class: parent: marello_magento2.import_strategy.default_magento2_import + calls: + - ['setEnableCollectingOriginIds', [true]] marello_magento2.import_strategy.attributeset: parent: marello_magento2.import_strategy.default_magento2_import @@ -158,17 +164,12 @@ services: calls: - ['setActionName', [!php/const Marello\Bundle\Magento2Bundle\Integration\Connector\OrderConnector::EXPORT_ACTION_UPDATE_ORDER_STATUS]] - oro_importexport.reader.entity: - class: Oro\Bundle\ImportExportBundle\Reader\EntityReader + marello_magento2.importexport.reader.origin_ids_context: + class: Marello\Bundle\Magento2Bundle\ImportExport\Reader\OriginIdsContextReader arguments: - '@oro_importexport.context_registry' - - '@doctrine' - - '@oro_security.owner.ownership_metadata_provider' calls: - - [setDispatcher, ['@event_dispatcher']] - - [setAclHelper, ['@oro_security.acl_helper']] - tags: - - { name: oro_importexport.reader, alias: entity } + - ['setLogger', ['@oro_integration.logger.strategy']] # Writers marello_magento2.importexport.writer.export.abstract: @@ -211,6 +212,19 @@ services: class: Marello\Bundle\Magento2Bundle\ImportExport\Writer\OrderExportUpdateStatusWriter parent: marello_magento2.importexport.writer.export.abstract + # Remover + marello_magento2.importexport.remover.non_existed_records: + class: Marello\Bundle\Magento2Bundle\ImportExport\Remover\NonExistedRecordsRemover + arguments: + - '@doctrine' + - '@oro_importexport.context_registry' + calls: + - ['setLogger', ['@oro_integration.logger.strategy']] + + marello_magento2.importexport.remover.website_mapping_records: + class: Marello\Bundle\Magento2Bundle\ImportExport\Remover\WebsiteMappingRecordsRemover + parent: marello_magento2.importexport.remover.non_existed_records + # Data converters marello_magento2.importexport.converter.website: class: Marello\Bundle\Magento2Bundle\ImportExport\Converter\WebsiteDataConverter @@ -343,6 +357,9 @@ services: marello_magento2.importexport.processor.export.product_delete_removed_action: class: Akeneo\Bundle\BatchBundle\Item\Support\NoopProcessor + marello_magento2.importexport.processor.origin_ids_context: + class: Akeneo\Bundle\BatchBundle\Item\Support\NoopProcessor + # Sync Processors marello_magento2.integration.sync_processor.abstract: class: Marello\Bundle\Magento2Bundle\Integration\SyncProcessor\AbstractSyncProcessor diff --git a/src/Marello/Bundle/Magento2Bundle/Resources/config/oro/assets.yml b/src/Marello/Bundle/Magento2Bundle/Resources/config/oro/assets.yml new file mode 100644 index 000000000..bc90f48f9 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/Resources/config/oro/assets.yml @@ -0,0 +1,3 @@ +css: + inputs: + - 'bundles/marellomagento2/css/scss/main.scss' diff --git a/src/Marello/Bundle/Magento2Bundle/Resources/config/services.yml b/src/Marello/Bundle/Magento2Bundle/Resources/config/services.yml index 3df12fed5..7c0a87719 100644 --- a/src/Marello/Bundle/Magento2Bundle/Resources/config/services.yml +++ b/src/Marello/Bundle/Magento2Bundle/Resources/config/services.yml @@ -17,6 +17,9 @@ services: marello_magento2.transport.rest.request.request_factory: class: Marello\Bundle\Magento2Bundle\Transport\Rest\Request\RequestFactory + marello_magento2.transport.rest.request.shifted_items_search_request_factory: + class: Marello\Bundle\Magento2Bundle\Transport\Rest\Request\ShiftedItemsSearchRequestFactory + marello_magento2.transport.rest.search_client_factory: class: Marello\Bundle\Magento2Bundle\Transport\Rest\Client\SearchClientFactory @@ -58,6 +61,12 @@ services: arguments: - 'Marello\Bundle\Magento2Bundle\Entity\Store' + marello_magento2.repository.product_tax_class: + class: Marello\Bundle\Magento2Bundle\Entity\Repository\ProductTaxClassRepository + parent: oro_entity.abstract_repository + arguments: + - 'Marello\Bundle\Magento2Bundle\Entity\ProductTaxClass' + # Event Listeners marello_magento2.event_listener.doctrine.product: class: Marello\Bundle\Magento2Bundle\EventListener\Doctrine\ProductListener @@ -96,6 +105,7 @@ services: - '@marello_magento2.stack.product_changes_by_channel' - '@marello_product.repository.product' - '@oro_message_queue.message_producer' + - '@oro_integration.genuine_sync_scheduler' tags: - { name: doctrine.event_listener, event: onFlush } - { name: doctrine.event_listener, event: postFlush } @@ -155,6 +165,16 @@ services: - { name: doctrine.event_listener, event: postFlush } - { name: doctrine.event_listener, event: onClear } + marello_magento2.event_listener.doctrine.magento2_transport: + class: Marello\Bundle\Magento2Bundle\EventListener\Doctrine\Magento2TransportListener + arguments: + - '@oro_integration.genuine_sync_scheduler' + - ['initialSyncStartDate', 'websiteToSalesChannelMapping'] + tags: + - { name: doctrine.orm.entity_listener, entity: 'Marello\Bundle\Magento2Bundle\Entity\Magento2Transport', event: postUpdate } + - { name: doctrine.event_listener, event: postFlush } + - { name: doctrine.event_listener, event: onClear } + # Handlers marello_magento2.handler.transport_entity_handler: class: Marello\Bundle\Magento2Bundle\Handler\TransportEntityHandler @@ -169,6 +189,7 @@ services: - '@marello_magento2.transport.rest.search_client_factory' - '@oro_security.encoder.default' - '@marello_magento2.transport.rest.search_criteria.filter_factory' + - '@marello_magento2.transport.rest.request.shifted_items_search_request_factory' calls: - [setLogger, ["@oro_integration.logger.strategy"]] tags: @@ -191,8 +212,6 @@ services: marello_magento2.provider.websites_provider: class: Marello\Bundle\Magento2Bundle\Provider\WebsitesProvider - arguments: - - '@translator' marello_magento2.provider.tracked_sales_channel: class: Marello\Bundle\Magento2Bundle\Provider\TrackedSalesChannelProvider diff --git a/src/Marello/Bundle/Magento2Bundle/Resources/public/css/scss/main.scss b/src/Marello/Bundle/Magento2Bundle/Resources/public/css/scss/main.scss new file mode 100644 index 000000000..502604ed2 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/Resources/public/css/scss/main.scss @@ -0,0 +1,3 @@ +.website-to-sales-channel-hidden-group { + padding-left: 282px; +} diff --git a/src/Marello/Bundle/Magento2Bundle/Resources/views/Form/fields.html.twig b/src/Marello/Bundle/Magento2Bundle/Resources/views/Form/fields.html.twig index d1a5dc65b..f60169962 100644 --- a/src/Marello/Bundle/Magento2Bundle/Resources/views/Form/fields.html.twig +++ b/src/Marello/Bundle/Magento2Bundle/Resources/views/Form/fields.html.twig @@ -72,8 +72,7 @@ {% endblock %} {% block marello_magento2_website_to_sales_channel_widget %} - {# @todo: Move this scss file #} -
+
{{ block('hidden_widget') }}
{% endblock %} diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClient.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClient.php index ba4562134..ace37bcf7 100644 --- a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClient.php +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClient.php @@ -9,30 +9,14 @@ class SearchClient { - /** @var int */ - private const DEFAULT_CROSS_ITEM_PERCENT = 20; - /** @var RestClientInterface */ protected $innerClient; /** * @param RestClientInterface $innerClient - * @param int|null $crossItemPercentPerPage */ - public function __construct( - RestClientInterface $innerClient, - int $crossItemPercentPerPage = null - ) { - /** - * @todo Think about how to make the functionality, - * that fixes the issue with data shifting between the page - * f.e. when some record that we see on 1 current page was removed while we reading this page - * when we call the 2nd page the 1st record from this page was the 2nd one before the record has been removed - */ - if (null === $crossItemPercentPerPage) { - $crossItemCountPerPage = self::DEFAULT_CROSS_ITEM_PERCENT; - } - + public function __construct(RestClientInterface $innerClient) + { $this->innerClient = $innerClient; } diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClientFactory.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClientFactory.php index 4b448cf58..965d42ea8 100644 --- a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClientFactory.php +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Client/SearchClientFactory.php @@ -8,11 +8,10 @@ class SearchClientFactory { /** * @param RestClientInterface $client - * @param int|null $crossItemPercentPerPage * @return SearchClient */ - public function createSearchClient(RestClientInterface $client, int $crossItemPercentPerPage = null): SearchClient + public function createSearchClient(RestClientInterface $client): SearchClient { - return new SearchClient($client, $crossItemPercentPerPage); + return new SearchClient($client); } } diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchIterator.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchIterator.php index 2ec7af0a6..58362298d 100644 --- a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchIterator.php +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchIterator.php @@ -195,11 +195,21 @@ protected function loadNextPage() } $searchResponse = $this->loadPage(); - $this->firstLoaded = true; - $this->rows = $searchResponse->getItems(); - $this->totalCount = $searchResponse->getTotalCount(); - $this->offset = 0; + $this->processSearchResponseDTO($searchResponse); return count($this->rows) > 0; } + + /** + * Fill properties with information from the new page + * + * @param SearchResponseDTO $searchResponseDTO + */ + protected function processSearchResponseDTO(SearchResponseDTO $searchResponseDTO): void + { + $this->firstLoaded = true; + $this->rows = $searchResponseDTO->getItems(); + $this->totalCount = $searchResponseDTO->getTotalCount(); + $this->offset = 0; + } } diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchWithShiftCheckingIterator.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchWithShiftCheckingIterator.php new file mode 100644 index 000000000..b40fd7d49 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/AbstractSearchWithShiftCheckingIterator.php @@ -0,0 +1,134 @@ +shiftedItemsSearchRequestFactory = $shiftedItemsSearchRequestFactory; + } + + /** + * Allow to check data items not shifted between the page, + * this can occurs when item was removed or updated and you use field "updated_at" to filter records. + * In this case the updated records goes to the end of the item list and all records that located + * after updated item, change its position on minus one. In result the 1st item from the next page, + * goes to the place of last item of the current page. And you won't get it within current sync process. + * + * {@inheritDoc} + */ + protected function processSearchResponseDTO(SearchResponseDTO $searchResponseDTO): void + { + $shiftedItems = []; + if ($this->doNeedToCheckOnExistenceOfShiftedItems($searchResponseDTO)) { + $shiftedItems = $this->getShiftedItems($searchResponseDTO); + } + + parent::processSearchResponseDTO($searchResponseDTO); + + if (!empty($shiftedItems)) { + $this->logger->info( + '[Magento 2] Found shifted items, it will be added to the current row set.', + [ + 'shiftedItemsCount' => count($shiftedItems) + ] + ); + + \array_unshift($this->rows, ...$shiftedItems); + /** + * Move position on 2 element per one shifted, + * because we need take into account that updated element and shifted elements. + */ + $this->position -= count($shiftedItems) * 2; + } else { + $this->logger->info( + '[Magento 2] Shifted items hasn\'t found. Continue loading next page.' + ); + } + } + + /** + * @param SearchResponseDTO $searchResponseDTO + * @return bool + */ + protected function doNeedToCheckOnExistenceOfShiftedItems(SearchResponseDTO $searchResponseDTO): bool + { + if (false === $this->firstLoaded) { + return false; + } + + $countOfShiftedElement = $this->totalCount - $searchResponseDTO->getTotalCount(); + return $countOfShiftedElement > 0; + } + + /** + * @param SearchResponseDTO $searchResponseDTO + * @return array + */ + protected function getShiftedItems(SearchResponseDTO $searchResponseDTO): array + { + $countOfShiftedElements = $this->totalCount - $searchResponseDTO->getTotalCount(); + $searchRequest = $this->shiftedItemsSearchRequestFactory->getSearchRequest( + $this->searchRequest, + $countOfShiftedElements + ); + + $this->logger->info( + '[Magento 2] Check shifted items within previous search request.', + $searchRequest->getSearchCriteria()->getSearchCriteriaParams() + ); + + $searchResponse = $this->searchClient->search($searchRequest); + $items = $searchResponse->getItems(); + + if (empty($items)) { + return []; + } + + $shiftedItems = []; + $idColumnName = $this->getIdColumnNameToCompareReadItems(); + $itemsToCheckReverse = \array_reverse($items, false); + $existedItemsReverse = \array_reverse($this->rows, false); + foreach ($itemsToCheckReverse as $index => $itemToCheck) { + $existedItemIdValue = $existedItemsReverse[$index][$idColumnName] ?? null; + $itemToCheckIdValue = $itemsToCheckReverse[$index][$idColumnName] ?? null; + if (null === $existedItemIdValue || null === $itemToCheckIdValue) { + break; + } + + if ($existedItemIdValue !== $itemToCheckIdValue) { + $shiftedItems[] = $itemToCheck; + } else { + break; + } + } + + return $shiftedItems; + } + + /** + * @return string + */ + abstract protected function getIdColumnNameToCompareReadItems(): string; +} diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/OrderIterator.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/OrderIterator.php index 85f7fdcf2..e2f73ded0 100644 --- a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/OrderIterator.php +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Iterator/OrderIterator.php @@ -11,14 +11,12 @@ use Marello\Bundle\Magento2Bundle\Transport\Rest\SearchCriteria\SortOrder; use Oro\Bundle\IntegrationBundle\Provider\Rest\Exception\RestException; -class OrderIterator extends AbstractSearchIterator implements UpdatableSearchLoaderInterface +class OrderIterator extends AbstractSearchWithShiftCheckingIterator implements UpdatableSearchLoaderInterface { /** @var int */ public const DEFAULT_PAGE_SIZE = 10; - /** - * @var SearchParametersDTO - */ + /** @var SearchParametersDTO */ protected $searchParametersDTO; /** @@ -109,6 +107,23 @@ protected function initSearchCriteria(): void ); } + /** + * {@inheritDoc} + */ + protected function getIdColumnNameToCompareReadItems(): string + { + return MagentoOrderDataConverter::ID_COLUMN_NAME; + } + + /** + * {@inheritDoc} + */ + protected function doNeedToCheckOnExistenceOfShiftedItems(SearchResponseDTO $searchResponseDTO): bool + { + return parent::doNeedToCheckOnExistenceOfShiftedItems($searchResponseDTO) && + false === $this->searchParametersDTO->isInitialMode(); + } + /** * @return string */ diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/SearchRequest.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/SearchRequest.php index 11e84d9b0..b61239166 100644 --- a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/SearchRequest.php +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/SearchRequest.php @@ -63,4 +63,9 @@ protected function tryInitDefaultSearchCriteria(): self return $this; } + + public function __clone() + { + $this->searchCriteria = clone $this->searchCriteria; + } } diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactory.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactory.php new file mode 100644 index 000000000..4711c0da2 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactory.php @@ -0,0 +1,34 @@ +getSearchCriteria()->getPageSize(); + $previousPageNumber = $currentSearchRequest->getSearchCriteria()->getCurrentPage() - 1; + + $increment = -1; + $newPageNumber = $previousPageNumber; + $possiblePageSize = $countOfShiftedElements < $currentPageSize ? $countOfShiftedElements : $currentPageSize; + + do { + $increment++; + $newPageSize = $possiblePageSize + $increment; + if ($currentPageSize % $newPageSize === 0) { + $newPageNumber = $currentPageSize / $newPageSize * $previousPageNumber; + break; + } + } while($possiblePageSize < $currentPageSize); + + $newSearchRequest->getSearchCriteria()->setPageSize($newPageSize); + $newSearchRequest->getSearchCriteria()->setCurrentPage($newPageNumber); + + return $newSearchRequest; + } +} diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactoryInterface.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactoryInterface.php new file mode 100644 index 000000000..031e947d4 --- /dev/null +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/Request/ShiftedItemsSearchRequestFactoryInterface.php @@ -0,0 +1,16 @@ +format($format); -// } -// -// public function -} diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/SearchCriteria/SearchCriteria.php b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/SearchCriteria/SearchCriteria.php index ac32f0889..b52bfe002 100644 --- a/src/Marello/Bundle/Magento2Bundle/Transport/Rest/SearchCriteria/SearchCriteria.php +++ b/src/Marello/Bundle/Magento2Bundle/Transport/Rest/SearchCriteria/SearchCriteria.php @@ -49,6 +49,14 @@ public function setCurrentPage(int $currentPage): self return $this; } + /** + * @return int + */ + public function getCurrentPage(): int + { + return $this->currentPage; + } + /** * @return $this */ @@ -70,6 +78,14 @@ public function setPageSize(int $pageSize): self return $this; } + /** + * @return int + */ + public function getPageSize(): int + { + return $this->pageSize; + } + /** * @param Filter $filter * @param Filter|null $orFilter diff --git a/src/Marello/Bundle/Magento2Bundle/Transport/RestTransport.php b/src/Marello/Bundle/Magento2Bundle/Transport/RestTransport.php index e97fee51a..c19fa2df4 100644 --- a/src/Marello/Bundle/Magento2Bundle/Transport/RestTransport.php +++ b/src/Marello/Bundle/Magento2Bundle/Transport/RestTransport.php @@ -13,6 +13,7 @@ use Marello\Bundle\Magento2Bundle\Transport\Rest\Iterator\StoreIterator; use Marello\Bundle\Magento2Bundle\Transport\Rest\Iterator\WebsiteIterator; use Marello\Bundle\Magento2Bundle\Transport\Rest\Request\RequestFactory; +use Marello\Bundle\Magento2Bundle\Transport\Rest\Request\ShiftedItemsSearchRequestFactoryInterface; use Marello\Bundle\Magento2Bundle\Transport\Rest\SearchCriteria\FilterFactoryInterface; use Oro\Bundle\IntegrationBundle\Entity\Transport; use Oro\Bundle\IntegrationBundle\Provider\Rest\Client\FactoryInterface as RestClientFactoryInterface; @@ -71,25 +72,33 @@ class RestTransport implements Magento2TransportInterface, LoggerAwareInterface */ protected $filterFactory; + /** + * @var ShiftedItemsSearchRequestFactoryInterface + */ + protected $shiftedItemsSearchRequestFactory; + /** * @param RestClientFactoryInterface $clientFactory * @param RequestFactory $requestFactory * @param SearchClientFactory $searchClientFactory * @param SymmetricCrypterInterface $crypter * @param FilterFactoryInterface $filterFactory + * @param ShiftedItemsSearchRequestFactoryInterface $shiftedItemsSearchRequestFactory */ public function __construct( RestClientFactoryInterface $clientFactory, RequestFactory $requestFactory, SearchClientFactory $searchClientFactory, SymmetricCrypterInterface $crypter, - FilterFactoryInterface $filterFactory + FilterFactoryInterface $filterFactory, + ShiftedItemsSearchRequestFactoryInterface $shiftedItemsSearchRequestFactory ) { $this->clientFactory = $clientFactory; $this->requestFactory = $requestFactory; $this->searchClientFactory = $searchClientFactory; $this->crypter = $crypter; $this->filterFactory = $filterFactory; + $this->shiftedItemsSearchRequestFactory = $shiftedItemsSearchRequestFactory; } /** @@ -264,7 +273,12 @@ public function getOrders(): \Iterator $searchClient = $this->searchClientFactory->createSearchClient($this->getClient()); - return new OrderIterator($searchClient, $request, $this->filterFactory); + return new OrderIterator( + $searchClient, + $request, + $this->filterFactory, + $this->shiftedItemsSearchRequestFactory + ); } /**