From 4f6e22790fcd21d36c857f45cd37178c200f8d5e Mon Sep 17 00:00:00 2001 From: Johan Liefers Date: Thu, 7 Jan 2016 15:03:20 +0100 Subject: [PATCH 01/48] added jsontype --- Resources/config/types.xml | 5 ++++ Solr/Type/JsonType.php | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 Solr/Type/JsonType.php diff --git a/Resources/config/types.xml b/Resources/config/types.xml index d2de6bc..057d248 100644 --- a/Resources/config/types.xml +++ b/Resources/config/types.xml @@ -12,6 +12,7 @@ Integrated\Bundle\SolrBundle\Solr\Type\FieldAppendMapperType Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType Integrated\Bundle\SolrBundle\Solr\Type\RemoveType + Integrated\Bundle\SolrBundle\Solr\Type\JsonType @@ -41,6 +42,10 @@ + + + + \ No newline at end of file diff --git a/Solr/Type/JsonType.php b/Solr/Type/JsonType.php new file mode 100644 index 0000000..e2ee82c --- /dev/null +++ b/Solr/Type/JsonType.php @@ -0,0 +1,59 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Solr\Type; + +use Symfony\Component\PropertyAccess\PropertyAccess; + +use Integrated\Bundle\ContentBundle\Document\Content\Content; +use Integrated\Common\Converter\ContainerInterface; +use Integrated\Common\Converter\Type\TypeInterface; + +/** + * @author Johan Liefers + */ +class JsonType implements TypeInterface +{ + /** + * {@inheritdoc} + */ + public function build(ContainerInterface $container, $data, array $options = []) + { + if (!($data instanceof Content)) { + return; + } + $relations = $data->getReferencesByRelationId($options['relation_id']); + + $accessor = PropertyAccess::createPropertyAccessor(); + + $array = []; + $i = 0; + foreach ($relations as $relation) { + foreach ($options['properties'] as $alias => $property) { + $array[$i][$alias] = $accessor->getValue($relation, $property); + } + $i++; + } + + $container->set( + $options['alias'], + json_encode($array) + ); + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'integrated.json'; + } +} From 06d1bc43b7a5156c003e717800a4a0030b987981 Mon Sep 17 00:00:00 2001 From: Johan Liefers Date: Mon, 11 Jan 2016 15:08:58 +0100 Subject: [PATCH 02/48] renamed to relation_json --- Resources/config/types.xml | 4 ++-- Solr/Type/{JsonType.php => RelationJsonType.php} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename Solr/Type/{JsonType.php => RelationJsonType.php} (93%) diff --git a/Resources/config/types.xml b/Resources/config/types.xml index 057d248..fea3ed1 100644 --- a/Resources/config/types.xml +++ b/Resources/config/types.xml @@ -12,7 +12,7 @@ Integrated\Bundle\SolrBundle\Solr\Type\FieldAppendMapperType Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType Integrated\Bundle\SolrBundle\Solr\Type\RemoveType - Integrated\Bundle\SolrBundle\Solr\Type\JsonType + Integrated\Bundle\SolrBundle\Solr\Type\RelationJsonType @@ -42,7 +42,7 @@ - + diff --git a/Solr/Type/JsonType.php b/Solr/Type/RelationJsonType.php similarity index 93% rename from Solr/Type/JsonType.php rename to Solr/Type/RelationJsonType.php index e2ee82c..d1a3233 100644 --- a/Solr/Type/JsonType.php +++ b/Solr/Type/RelationJsonType.php @@ -20,7 +20,7 @@ /** * @author Johan Liefers */ -class JsonType implements TypeInterface +class RelationJsonType implements TypeInterface { /** * {@inheritdoc} @@ -54,6 +54,6 @@ public function build(ContainerInterface $container, $data, array $options = []) */ public function getName() { - return 'integrated.json'; + return 'integrated.relation_json'; } } From b06c2da97d088b4d1dc8a7d11a487829cdb93a48 Mon Sep 17 00:00:00 2001 From: Johan Liefers Date: Mon, 11 Jan 2016 15:13:28 +0100 Subject: [PATCH 03/48] removed count $i --- Solr/Type/RelationJsonType.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Solr/Type/RelationJsonType.php b/Solr/Type/RelationJsonType.php index d1a3233..b4ee283 100644 --- a/Solr/Type/RelationJsonType.php +++ b/Solr/Type/RelationJsonType.php @@ -35,12 +35,10 @@ public function build(ContainerInterface $container, $data, array $options = []) $accessor = PropertyAccess::createPropertyAccessor(); $array = []; - $i = 0; - foreach ($relations as $relation) { + foreach ($relations as $key => $relation) { foreach ($options['properties'] as $alias => $property) { - $array[$i][$alias] = $accessor->getValue($relation, $property); + $array[$key][$alias] = $accessor->getValue($relation, $property); } - $i++; } $container->set( From fd12e08b5ca464ecf4bea4e31dd8fad0e6538bdd Mon Sep 17 00:00:00 2001 From: Johan Liefers Date: Tue, 19 Jan 2016 12:46:53 +0100 Subject: [PATCH 04/48] added solr property type --- Resources/config/types.xml | 5 ++++ Solr/Type/PropertyType.php | 56 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 Solr/Type/PropertyType.php diff --git a/Resources/config/types.xml b/Resources/config/types.xml index fea3ed1..eea5cb7 100644 --- a/Resources/config/types.xml +++ b/Resources/config/types.xml @@ -13,6 +13,7 @@ Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType Integrated\Bundle\SolrBundle\Solr\Type\RemoveType Integrated\Bundle\SolrBundle\Solr\Type\RelationJsonType + Integrated\Bundle\SolrBundle\Solr\Type\PropertyType @@ -46,6 +47,10 @@ + + + + \ No newline at end of file diff --git a/Solr/Type/PropertyType.php b/Solr/Type/PropertyType.php new file mode 100644 index 0000000..7c11219 --- /dev/null +++ b/Solr/Type/PropertyType.php @@ -0,0 +1,56 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Solr\Type; + +use Symfony\Component\PropertyAccess\PropertyAccess; + +use Integrated\Common\Content\ContentInterface; +use Integrated\Common\Converter\ContainerInterface; +use Integrated\Common\Converter\Type\TypeInterface; + +/** + * @author Johan Liefers + */ +class PropertyType implements TypeInterface +{ + /** + * {@inheritdoc} + */ + public function build(ContainerInterface $container, $data, array $options = []) + { + if (!($data instanceof ContentInterface)) { + return; + } + + $accessor = PropertyAccess::createPropertyAccessor(); + + foreach ($options as $condition) { + if (isset($condition['fieldValue'])) { + if ($condition['fieldValue'] == $accessor->getValue($data, $condition['field'])) { + $container->add('facet_properties', $condition['label']); + } + } elseif (isset($condition['fieldValueNot'])) { + if ($condition['fieldValueNot'] != $accessor->getValue($data, $condition['field'])) { + $container->add('facet_properties', $condition['label']); + } + } + } + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'integrated.properties'; + } +} From 6a48a52e85306f5b55864a2cd9ac3525a2b4bdad Mon Sep 17 00:00:00 2001 From: Ger Jan Date: Fri, 6 May 2016 12:25:56 +0200 Subject: [PATCH 05/48] [INTEGRATED-802] Added JSON Solr mapping type --- Resources/config/types.xml | 7 ++- Solr/Type/FieldMapperType.php | 46 ++++++++++------- Solr/Type/JsonType.php | 95 +++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 20 deletions(-) create mode 100644 Solr/Type/JsonType.php diff --git a/Resources/config/types.xml b/Resources/config/types.xml index eea5cb7..a3d9996 100644 --- a/Resources/config/types.xml +++ b/Resources/config/types.xml @@ -14,6 +14,7 @@ Integrated\Bundle\SolrBundle\Solr\Type\RemoveType Integrated\Bundle\SolrBundle\Solr\Type\RelationJsonType Integrated\Bundle\SolrBundle\Solr\Type\PropertyType + Integrated\Bundle\SolrBundle\Solr\Type\JsonType @@ -51,6 +52,10 @@ + + + + - \ No newline at end of file + diff --git a/Solr/Type/FieldMapperType.php b/Solr/Type/FieldMapperType.php index 2769703..90a0643 100644 --- a/Solr/Type/FieldMapperType.php +++ b/Solr/Type/FieldMapperType.php @@ -39,7 +39,7 @@ class FieldMapperType implements TypeInterface /** * @var PropertyAccessorInterface */ - private $accessor; + protected $accessor; /** * Constructor. @@ -56,6 +56,29 @@ public function __construct(PropertyAccessorInterface $accessor = null) * {@inheritdoc} */ public function build(ContainerInterface $container, $data, array $options = []) + { + foreach ($this->groupFields($options) as $field => $config) { + $this->remove($container, $field); + + foreach ($this->read($data, $config) as $value) { + $this->append($container, $field, $value); + } + } + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'integrated.fields'; + } + + /** + * @param array $options + * @return array + */ + protected function groupFields(array $options = []) { $fields = []; @@ -69,26 +92,12 @@ public function build(ContainerInterface $container, $data, array $options = []) $value = ['@' . $value]; // convert simple config to advance config } - foreach ($value as $config) { - $fields[$field][] = $config; + foreach ($value as $key => $config) { + $fields[$field][$key] = $config; } } - foreach ($fields as $field => $config) { - $this->remove($container, $field); - - foreach ($this->read($data, $config) as $value) { - $this->append($container, $field, $value); - } - } - } - - /** - * {@inheritdoc} - */ - public function getName() - { - return 'integrated.fields'; + return $fields; } /** @@ -159,7 +168,6 @@ protected function readArray($data, array $paths, $separator = ' ') foreach ($paths as $index => $path) { if (is_array($path)) { - // Since $path is a array the $index with be treated as a path and the result of that // path is treated as a array. If the result is not a array then it will be placed in // a array to simulate that the result is a array. diff --git a/Solr/Type/JsonType.php b/Solr/Type/JsonType.php new file mode 100644 index 0000000..bce51f2 --- /dev/null +++ b/Solr/Type/JsonType.php @@ -0,0 +1,95 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Solr\Type; + +use Symfony\Component\PropertyAccess\Exception\ExceptionInterface; + +use Integrated\Common\Converter\ContainerInterface; + +use Traversable; + +/** + * @author Ger Jan van den Bosch + */ +class JsonType extends FieldMapperType +{ + /** + * {@inheritdoc} + */ + public function build(ContainerInterface $container, $data, array $options = []) + { + foreach ($this->groupFields($options) as $field => $config) { + $this->remove($container, $field); + + if (is_array($config)) { + foreach ($this->readValues($data, $config) as $values) { + foreach ($values as $value) { + $this->append($container, $field, json_encode($value)); + } + } + } + } + } + + /** + * @param mixed $data + * @param array $paths + * + * @return array + */ + protected function readValues($data, array $paths) + { + $extracted = []; + + foreach ($paths as $index => $path) { + $index = (string)$index; + + if (is_array($path)) { + try { + $array = $this->accessor->getValue($data, $index); + + if (!is_array($array) && !$array instanceof Traversable) { + $array = [$array]; + } + } catch (ExceptionInterface $e) { + $array = []; + } + + $results = []; + + foreach ($array as $value) { + if ($path) { + $results[] = $this->readValues($value, $path); + } else { + if ($value = $this->convert($value)) { + $results[] = $value; + } + } + } + + $extracted[$index] = $results; + } else { + $extracted[$index] = $this->readString($data, $path); + } + } + + return $extracted; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'integrated.json'; + } +} From 8fb1c2bcd325f5149267b108094e1d9c45db35ad Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Wed, 18 May 2016 12:13:49 +0200 Subject: [PATCH 06/48] INTEGRATED-669 errors while converting document will no longer halt the indexer. - Console will output error in case of a full or daemon indexer run. - Update the bundle to work with the latest (0.5) library version. - Indexing error are now catched and logged in de channel solr. --- Command/IndexerRunCommand.php | 5 +- .../IntegratedSolrExtension.php | 1 + EventListener/ErrorLogger.php | 68 +++++++++++++++++++ IntegratedSolrBundle.php | 2 + Resources/config/event.xml | 15 ++++ Resources/config/indexer.xml | 45 ++++++------ composer.json | 2 +- 7 files changed, 114 insertions(+), 24 deletions(-) create mode 100644 EventListener/ErrorLogger.php create mode 100644 Resources/config/event.xml diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index 2ce44fc..d6c19d9 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -115,7 +115,10 @@ private function runExternal(InputInterface $input, OutputInterface $output) 'php app/console solr:indexer:run -e ' . $input->getOption('env'), $this->getRootDir() ); - $process->run(); + + $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer, false, $type); + }); if (!$process->isSuccessful()) { break; // terminate when there is a error diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 0df92e7..4021510 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -38,6 +38,7 @@ public function load(array $configs, ContainerBuilder $container) $loader = new XmlFileLoader($container, new FileLocator(__DIR__ . '/../Resources/config')); $loader->load('converter.xml'); + $loader->load('event.xml'); $loader->load('indexer.xml'); $loader->load('queue.xml'); $loader->load('solarium.xml'); diff --git a/EventListener/ErrorLogger.php b/EventListener/ErrorLogger.php new file mode 100644 index 0000000..45f4e20 --- /dev/null +++ b/EventListener/ErrorLogger.php @@ -0,0 +1,68 @@ + +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +namespace Integrated\Bundle\SolrBundle\EventListener; + +use Psr\Log\LoggerInterface; + +use Integrated\Common\Solr\Indexer\Event\ErrorEvent; +use Integrated\Common\Solr\Indexer\Events; +use Integrated\Common\Solr\Indexer\Job; + +use Symfony\Component\EventDispatcher\EventSubscriberInterface; + +/** + * @author Jan Sanne Mulder + */ +class ErrorLogger implements EventSubscriberInterface +{ + /** + * @var LoggerInterface + */ + private $logger = null; + + /** + * {@inheritdoc} + */ + public static function getSubscribedEvents() + { + return [ + Events::ERROR => 'onError' + ]; + } + + /** + * @param ErrorEvent $event + */ + public function onError(ErrorEvent $event) + { + if (null === $this->logger) { + return; + } + + $payload = $event->getMessage()->getPayload(); + + if ($payload instanceof Job) { + $this->logger->error($event->getException()->getMessage(), [ + 'action' => $payload->getAction(), + 'options' => $payload->getOptions() + ]); + } + } + + /** + * @param LoggerInterface $logger + */ + public function setLogger(LoggerInterface $logger = null) + { + $this->logger = $logger; + } +} diff --git a/IntegratedSolrBundle.php b/IntegratedSolrBundle.php index a42a3a8..497c7c3 100644 --- a/IntegratedSolrBundle.php +++ b/IntegratedSolrBundle.php @@ -15,6 +15,7 @@ use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTypePass; use Integrated\Bundle\SolrBundle\DependencyInjection\IntegratedSolrExtension; +use Symfony\Component\EventDispatcher\DependencyInjection\RegisterListenersPass; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\HttpKernel\Bundle\Bundle; @@ -32,6 +33,7 @@ public function build(ContainerBuilder $container) $container->addCompilerPass(new RegisterConfigFileProviderPass()); $container->addCompilerPass(new RegisterTypePass()); + $container->addCompilerPass(new RegisterListenersPass('integrated_solr.event.dispatcher', 'integrated_solr.event_listener', 'integrated_solr.event_subscriber')); } /** diff --git a/Resources/config/event.xml b/Resources/config/event.xml new file mode 100644 index 0000000..ce4119c --- /dev/null +++ b/Resources/config/event.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/Resources/config/indexer.xml b/Resources/config/indexer.xml index 10819c4..00ce17d 100644 --- a/Resources/config/indexer.xml +++ b/Resources/config/indexer.xml @@ -4,28 +4,25 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Integrated\MongoDB\Solr\Indexer\QueueSubscriber - - Symfony\Component\Serializer\Serializer - Integrated\MongoDB\Serializer\Normalizer\ContainerAwareDocumentNormalizer - Symfony\Component\Serializer\Encoder\JsonEncoder - - Integrated\Common\Solr\Indexer\Indexer + - + + + + - + + + - + - + @@ -34,25 +31,29 @@ - + doctrine.odm.mongodb.document_manager - + + + + + + - + + + - - - - - - + + + diff --git a/composer.json b/composer.json index e4b6c69..2061ddb 100644 --- a/composer.json +++ b/composer.json @@ -17,7 +17,7 @@ "php": ">=5.4", "symfony/symfony": "~2.4", "symfony/filesystem": "~2.6", - "integrated/library": "~0.3" + "integrated/library": "~0.5" }, "require-dev": { "squizlabs/php_codesniffer": "2.*", From f54dca1825ce87402f059508da3ce81315fd877b Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Thu, 16 Jun 2016 17:18:46 +0200 Subject: [PATCH 07/48] Implemented the solr worker services. - New task can be registered by tagging them as integrated_solr.task and adding the task class as attribute. - Errors are logged by monolog with in the channel solr. --- Command/WorkerCommand.php | 58 +++++++++++++++++ .../CompilerPass/RegisterTaskHandlerPass.php | 40 ++++++++++++ .../IntegratedSolrExtension.php | 2 + ...ErrorLogger.php => IndexerErrorLogger.php} | 2 +- EventListener/WorkerErrorLogger.php | 62 +++++++++++++++++++ IntegratedSolrBundle.php | 9 ++- Provider/MongoDBProvider.php | 53 ++++++++++++++++ Resources/config/indexer.xml | 11 +++- Resources/config/queue.xml | 18 ++++-- Resources/config/task.xml | 33 ++++++++++ Resources/config/worker.xml | 35 +++++++++++ 11 files changed, 312 insertions(+), 11 deletions(-) create mode 100644 Command/WorkerCommand.php create mode 100644 DependencyInjection/CompilerPass/RegisterTaskHandlerPass.php rename EventListener/{ErrorLogger.php => IndexerErrorLogger.php} (95%) create mode 100644 EventListener/WorkerErrorLogger.php create mode 100644 Provider/MongoDBProvider.php create mode 100644 Resources/config/task.xml create mode 100644 Resources/config/worker.xml diff --git a/Command/WorkerCommand.php b/Command/WorkerCommand.php new file mode 100644 index 0000000..d20f5df --- /dev/null +++ b/Command/WorkerCommand.php @@ -0,0 +1,58 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Command; + +use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; + +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Process\Process; +use Symfony\Component\Filesystem\LockHandler; + +/** + * @author Jan Sanne Mulder + */ +class WorkerCommand extends ContainerAwareCommand +{ + /** + * @see Command + */ + protected function configure() + { + $this + ->setName('solr:worker:run') + + ->addOption('tasks', 't', InputOption::VALUE_REQUIRED, 'The maximum number of tasks to execute in one worker run', null) + + ->setDescription('Execute worker task from the queue.') + ->setHelp(' +The %command.name% command starts a solr worker run. + +php %command.full_name% +'); + } + + /** + * {@inheritdoc} + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + $worker = $this->getContainer()->get('integrated_solr.worker'); + + if (null !== ($tasks = $input->getOption('tasks'))) { + $worker->setOption('tasks', intval($tasks)); + } + + $worker->execute(); + } +} diff --git a/DependencyInjection/CompilerPass/RegisterTaskHandlerPass.php b/DependencyInjection/CompilerPass/RegisterTaskHandlerPass.php new file mode 100644 index 0000000..0bd3eee --- /dev/null +++ b/DependencyInjection/CompilerPass/RegisterTaskHandlerPass.php @@ -0,0 +1,40 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass; + +use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\ContainerBuilder; +use Symfony\Component\DependencyInjection\Reference; + +/** + * @author Jan Sanne Mulder + */ +class RegisterTaskHandlerPass implements CompilerPassInterface +{ + /** + * {@inheritdoc} + */ + public function process(ContainerBuilder $container) + { + if (!$container->hasDefinition('integrated_solr.worker.handler.registry_builder')) { + return; + } + + $definition = $container->getDefinition('integrated_solr.worker.handler.registry_builder'); + + foreach ($container->findTaggedServiceIds('integrated_solr.task') as $service => $tags) { + foreach ($tags as $attributes) { + $definition->addMethodCall('addHandler', [$attributes['class'], new Reference($service)]); + } + } + } +} diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 4021510..5167731 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -42,7 +42,9 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('indexer.xml'); $loader->load('queue.xml'); $loader->load('solarium.xml'); + $loader->load('task.xml'); $loader->load('types.xml'); + $loader->load('worker.xml'); if ($container->getParameter('kernel.debug')) { $loader->load('collector.xml'); diff --git a/EventListener/ErrorLogger.php b/EventListener/IndexerErrorLogger.php similarity index 95% rename from EventListener/ErrorLogger.php rename to EventListener/IndexerErrorLogger.php index 45f4e20..bfcb29b 100644 --- a/EventListener/ErrorLogger.php +++ b/EventListener/IndexerErrorLogger.php @@ -22,7 +22,7 @@ /** * @author Jan Sanne Mulder */ -class ErrorLogger implements EventSubscriberInterface +class IndexerErrorLogger implements EventSubscriberInterface { /** * @var LoggerInterface diff --git a/EventListener/WorkerErrorLogger.php b/EventListener/WorkerErrorLogger.php new file mode 100644 index 0000000..8b85d92 --- /dev/null +++ b/EventListener/WorkerErrorLogger.php @@ -0,0 +1,62 @@ + +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +namespace Integrated\Bundle\SolrBundle\EventListener; + +use Psr\Log\LoggerInterface; + +use Integrated\Common\Solr\Task\Event\ErrorEvent; +use Integrated\Common\Solr\Task\Events; + +use Symfony\Component\EventDispatcher\EventSubscriberInterface; + +/** + * @author Jan Sanne Mulder + */ +class WorkerErrorLogger implements EventSubscriberInterface +{ + /** + * @var LoggerInterface + */ + private $logger = null; + + /** + * {@inheritdoc} + */ + public static function getSubscribedEvents() + { + return [ + Events::ERROR => 'onError' + ]; + } + + /** + * @param ErrorEvent $event + */ + public function onError(ErrorEvent $event) + { + if (null === $this->logger) { + return; + } + + $this->logger->error($event->getException()->getMessage(), [ + 'payload' => serialize($event->getMessage()->getPayload()) + ]); + } + + /** + * @param LoggerInterface $logger + */ + public function setLogger(LoggerInterface $logger = null) + { + $this->logger = $logger; + } +} diff --git a/IntegratedSolrBundle.php b/IntegratedSolrBundle.php index 497c7c3..cb2ebb6 100644 --- a/IntegratedSolrBundle.php +++ b/IntegratedSolrBundle.php @@ -12,6 +12,7 @@ namespace Integrated\Bundle\SolrBundle; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterConfigFileProviderPass; +use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTaskHandlerPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTypePass; use Integrated\Bundle\SolrBundle\DependencyInjection\IntegratedSolrExtension; @@ -33,7 +34,13 @@ public function build(ContainerBuilder $container) $container->addCompilerPass(new RegisterConfigFileProviderPass()); $container->addCompilerPass(new RegisterTypePass()); - $container->addCompilerPass(new RegisterListenersPass('integrated_solr.event.dispatcher', 'integrated_solr.event_listener', 'integrated_solr.event_subscriber')); + $container->addCompilerPass(new RegisterTaskHandlerPass()); + + $container->addCompilerPass(new RegisterListenersPass( + 'integrated_solr.event.dispatcher', + 'integrated_solr.event_listener', + 'integrated_solr.event_subscriber' + )); } /** diff --git a/Provider/MongoDBProvider.php b/Provider/MongoDBProvider.php new file mode 100644 index 0000000..e0b10f0 --- /dev/null +++ b/Provider/MongoDBProvider.php @@ -0,0 +1,53 @@ + +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +namespace Integrated\Bundle\SolrBundle\Provider; + +use Doctrine\ODM\MongoDB\DocumentRepository; +use Integrated\Common\Solr\Task\Provider\ContentProviderInterface; + +/** + * @author Jan Sanne Mulder + */ +class MongoDBProvider implements ContentProviderInterface +{ + /** + * @var DocumentRepository + */ + private $repository; + + /** + * Constructor. + * + * @param DocumentRepository $repository + */ + public function __construct(DocumentRepository $repository) + { + $this->repository = $repository; + } + + /** + * {@inheritdoc} + */ + public function getReferenced($id) + { + $iterator = $this->repository->createQueryBuilder() + ->field('relations.references.$id')->equals($id) + ->getQuery() + ->getIterator(); + + if (method_exists($iterator, 'timeout')) { + $iterator->timeout(-1); + } + + return $iterator; + } +} diff --git a/Resources/config/indexer.xml b/Resources/config/indexer.xml index 00ce17d..d9318d5 100644 --- a/Resources/config/indexer.xml +++ b/Resources/config/indexer.xml @@ -6,7 +6,7 @@ - + @@ -16,7 +16,7 @@ - + @@ -43,13 +43,18 @@ + + + json + + - + diff --git a/Resources/config/queue.xml b/Resources/config/queue.xml index bfc4394..8cc5921 100644 --- a/Resources/config/queue.xml +++ b/Resources/config/queue.xml @@ -49,12 +49,18 @@ - - solr-indexer - + + + solr-indexer + + + + + solr-worker + + + + diff --git a/Resources/config/task.xml b/Resources/config/task.xml new file mode 100644 index 0000000..d9515d3 --- /dev/null +++ b/Resources/config/task.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + Integrated\Bundle\ContentBundle\Document\Content\Content + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Resources/config/worker.xml b/Resources/config/worker.xml new file mode 100644 index 0000000..8143310 --- /dev/null +++ b/Resources/config/worker.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From fc5e53929bc2826729b6734486fb1509e4fdcfe1 Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Mon, 27 Jun 2016 10:22:17 +0200 Subject: [PATCH 08/48] Updated the readme with extra instructions. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index c839792..e4bc05b 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,13 @@ Adds support for Solr configuration and indexing of your documents or entities ## Documentation ## * [Integrated for developers website](http://www.integratedfordevelopers.com "Integrated for developers website") +### Commands ### +* The command **solr:indexer:queue** will queue up all the documents to be indexed +* The command **solr:indexer:run** will tranform the queued up documents to a solr compatible format and send them to solr for indexing. +* The command **solr:worker:run** will start a worker run and will process up to end of the queue or 1000 tasks, whichever comes first. + +It's recommended to the execute **solr:indexer:run** and **solr:worker:run** as automated tasks by configuring them as cronjobs. + ## Installation ## This bundle can be installed following these steps: @@ -26,6 +33,9 @@ This bundle can be installed following these steps: ); } +### Setup the queue ### +The solr bundle requires a queue to work properly, so if not already done setup the queue by executing the **init:queue** command. + ## License ## This bundle is under the MIT license. See the complete license in the bundle: From 0a6dd66e3eaa9b71edfe78797b3aa6fb605524fd Mon Sep 17 00:00:00 2001 From: Ger Jan Date: Tue, 30 Aug 2016 14:42:31 +0200 Subject: [PATCH 09/48] [INTEGRATED-869] Use FilterContainer by default --- Resources/config/converter.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/config/converter.xml b/Resources/config/converter.xml index 6aabb90..b713f6b 100644 --- a/Resources/config/converter.xml +++ b/Resources/config/converter.xml @@ -18,7 +18,7 @@ Integrated\Common\Converter\Config\ConfigResolver - Integrated\Common\Converter\ContainerFactory + Integrated\Common\Converter\FilterContainerFactory Integrated\Common\Converter\Converter @@ -56,4 +56,4 @@ - \ No newline at end of file + From e3d0fb6d630ebef88f80b48df513c2b0e05049d6 Mon Sep 17 00:00:00 2001 From: Ger Jan Date: Thu, 8 Sep 2016 17:01:01 +0200 Subject: [PATCH 10/48] [INTEGRATED-744] Moved PropertyType and RelationJsonType from SolrBundle to ContentBundle --- Resources/config/types.xml | 10 ------ Solr/Type/PropertyType.php | 56 --------------------------------- Solr/Type/RelationJsonType.php | 57 ---------------------------------- 3 files changed, 123 deletions(-) delete mode 100644 Solr/Type/PropertyType.php delete mode 100644 Solr/Type/RelationJsonType.php diff --git a/Resources/config/types.xml b/Resources/config/types.xml index a3d9996..11fccc4 100644 --- a/Resources/config/types.xml +++ b/Resources/config/types.xml @@ -12,8 +12,6 @@ Integrated\Bundle\SolrBundle\Solr\Type\FieldAppendMapperType Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType Integrated\Bundle\SolrBundle\Solr\Type\RemoveType - Integrated\Bundle\SolrBundle\Solr\Type\RelationJsonType - Integrated\Bundle\SolrBundle\Solr\Type\PropertyType Integrated\Bundle\SolrBundle\Solr\Type\JsonType @@ -44,14 +42,6 @@ - - - - - - - - diff --git a/Solr/Type/PropertyType.php b/Solr/Type/PropertyType.php deleted file mode 100644 index 7c11219..0000000 --- a/Solr/Type/PropertyType.php +++ /dev/null @@ -1,56 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Integrated\Bundle\SolrBundle\Solr\Type; - -use Symfony\Component\PropertyAccess\PropertyAccess; - -use Integrated\Common\Content\ContentInterface; -use Integrated\Common\Converter\ContainerInterface; -use Integrated\Common\Converter\Type\TypeInterface; - -/** - * @author Johan Liefers - */ -class PropertyType implements TypeInterface -{ - /** - * {@inheritdoc} - */ - public function build(ContainerInterface $container, $data, array $options = []) - { - if (!($data instanceof ContentInterface)) { - return; - } - - $accessor = PropertyAccess::createPropertyAccessor(); - - foreach ($options as $condition) { - if (isset($condition['fieldValue'])) { - if ($condition['fieldValue'] == $accessor->getValue($data, $condition['field'])) { - $container->add('facet_properties', $condition['label']); - } - } elseif (isset($condition['fieldValueNot'])) { - if ($condition['fieldValueNot'] != $accessor->getValue($data, $condition['field'])) { - $container->add('facet_properties', $condition['label']); - } - } - } - } - - /** - * {@inheritdoc} - */ - public function getName() - { - return 'integrated.properties'; - } -} diff --git a/Solr/Type/RelationJsonType.php b/Solr/Type/RelationJsonType.php deleted file mode 100644 index b4ee283..0000000 --- a/Solr/Type/RelationJsonType.php +++ /dev/null @@ -1,57 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Integrated\Bundle\SolrBundle\Solr\Type; - -use Symfony\Component\PropertyAccess\PropertyAccess; - -use Integrated\Bundle\ContentBundle\Document\Content\Content; -use Integrated\Common\Converter\ContainerInterface; -use Integrated\Common\Converter\Type\TypeInterface; - -/** - * @author Johan Liefers - */ -class RelationJsonType implements TypeInterface -{ - /** - * {@inheritdoc} - */ - public function build(ContainerInterface $container, $data, array $options = []) - { - if (!($data instanceof Content)) { - return; - } - $relations = $data->getReferencesByRelationId($options['relation_id']); - - $accessor = PropertyAccess::createPropertyAccessor(); - - $array = []; - foreach ($relations as $key => $relation) { - foreach ($options['properties'] as $alias => $property) { - $array[$key][$alias] = $accessor->getValue($relation, $property); - } - } - - $container->set( - $options['alias'], - json_encode($array) - ); - } - - /** - * {@inheritdoc} - */ - public function getName() - { - return 'integrated.relation_json'; - } -} From 3239d4fcd7ba1c059046b4bf6c7b2ec2b94ecad8 Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Thu, 15 Sep 2016 14:43:56 +0200 Subject: [PATCH 11/48] The mongo db provider now also implements the content type provider and registered the new content type queue task --- Provider/MongoDBProvider.php | 20 +++++++++++++++++++- Resources/config/task.xml | 15 +++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Provider/MongoDBProvider.php b/Provider/MongoDBProvider.php index e0b10f0..f5c98c8 100644 --- a/Provider/MongoDBProvider.php +++ b/Provider/MongoDBProvider.php @@ -13,11 +13,12 @@ use Doctrine\ODM\MongoDB\DocumentRepository; use Integrated\Common\Solr\Task\Provider\ContentProviderInterface; +use Integrated\Common\Solr\Task\Provider\ContentTypeProviderInterface; /** * @author Jan Sanne Mulder */ -class MongoDBProvider implements ContentProviderInterface +class MongoDBProvider implements ContentProviderInterface, ContentTypeProviderInterface { /** * @var DocumentRepository @@ -50,4 +51,21 @@ public function getReferenced($id) return $iterator; } + + /** + * {@inheritdoc} + */ + public function getContent($id) + { + $iterator = $this->repository->createQueryBuilder() + ->field('contentType')->equals($id) + ->getQuery() + ->getIterator(); + + if (method_exists($iterator, 'timeout')) { + $iterator->timeout(-1); + } + + return $iterator; + } } diff --git a/Resources/config/task.xml b/Resources/config/task.xml index d9515d3..831b9db 100644 --- a/Resources/config/task.xml +++ b/Resources/config/task.xml @@ -8,6 +8,13 @@ + + + + + + + @@ -28,6 +35,14 @@ + + + + + + + + \ No newline at end of file From 40741a8daf65bfe734bc746266438e6a9d1d5694 Mon Sep 17 00:00:00 2001 From: Marijn Otte Date: Sun, 30 Oct 2016 22:46:53 +0100 Subject: [PATCH 12/48] Update composer versions --- composer.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index 2061ddb..28ad986 100644 --- a/composer.json +++ b/composer.json @@ -15,9 +15,8 @@ "prefer-stable": true, "require": { "php": ">=5.4", - "symfony/symfony": "~2.4", - "symfony/filesystem": "~2.6", - "integrated/library": "~0.5" + "symfony/symfony": "~2.7", + "integrated/library": "~0.6" }, "require-dev": { "squizlabs/php_codesniffer": "2.*", From 0126bcb2483c146e6b57b40ad3a4d0a5113411d5 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Mon, 24 Oct 2016 17:06:00 +0200 Subject: [PATCH 13/48] Allow tags trough the DI. --- .../SolariumEventTagCompilerPass.php | 33 +++++++++++++++++++ IntegratedSolrBundle.php | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php diff --git a/DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php b/DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php new file mode 100644 index 0000000..1d03162 --- /dev/null +++ b/DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php @@ -0,0 +1,33 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass; + +use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\ContainerBuilder; +use Symfony\Component\DependencyInjection\Reference; + +/** + * @author Johnny Borg + */ +class SolariumEventTagCompilerPass implements CompilerPassInterface +{ + /** + * {@inheritdoc} + */ + public function process(ContainerBuilder $container) + { + $definition = $container->getDefinition('solarium.client'); + foreach ($container->findTaggedServiceIds('integrated_solr.event_subscriber') as $service => $tags) { + $definition ->addMethodCall('getEventDispatcher()->addSubscriber', [new Reference($service)]); + } + } +} diff --git a/IntegratedSolrBundle.php b/IntegratedSolrBundle.php index cb2ebb6..c44e7d0 100644 --- a/IntegratedSolrBundle.php +++ b/IntegratedSolrBundle.php @@ -14,6 +14,7 @@ use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterConfigFileProviderPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTaskHandlerPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTypePass; +use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\SolariumEventTagCompilerPass; use Integrated\Bundle\SolrBundle\DependencyInjection\IntegratedSolrExtension; use Symfony\Component\EventDispatcher\DependencyInjection\RegisterListenersPass; @@ -32,6 +33,7 @@ public function build(ContainerBuilder $container) { parent::build($container); + $container->addCompilerPass(new SolariumEventTagCompilerPass()); $container->addCompilerPass(new RegisterConfigFileProviderPass()); $container->addCompilerPass(new RegisterTypePass()); $container->addCompilerPass(new RegisterTaskHandlerPass()); From 7654f59eddf484bbe54b2d1a9c616d9950c65a41 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Wed, 9 Nov 2016 13:38:09 +0100 Subject: [PATCH 14/48] Fixed the unit tests. --- Solr/Type/FieldMapperType.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Solr/Type/FieldMapperType.php b/Solr/Type/FieldMapperType.php index 90a0643..18f2f2f 100644 --- a/Solr/Type/FieldMapperType.php +++ b/Solr/Type/FieldMapperType.php @@ -92,8 +92,8 @@ protected function groupFields(array $options = []) $value = ['@' . $value]; // convert simple config to advance config } - foreach ($value as $key => $config) { - $fields[$field][$key] = $config; + foreach ($value as $config) { + $fields[$field][] = $config; } } From 827953058439c8ab91846684e576479cc630954a Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Mon, 2 Jan 2017 13:36:58 +0100 Subject: [PATCH 15/48] Added Query expander --- Resources/config/solr.xml | 11 ++++++ Solr/Query/Expander.php | 44 ++++++++++++++++++++++ Tests/Solr/Query/ExpanderTest.php | 62 +++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 Resources/config/solr.xml create mode 100644 Solr/Query/Expander.php create mode 100644 Tests/Solr/Query/ExpanderTest.php diff --git a/Resources/config/solr.xml b/Resources/config/solr.xml new file mode 100644 index 0000000..62064c8 --- /dev/null +++ b/Resources/config/solr.xml @@ -0,0 +1,11 @@ + + + + + + + + + \ No newline at end of file diff --git a/Solr/Query/Expander.php b/Solr/Query/Expander.php new file mode 100644 index 0000000..5fcb5f7 --- /dev/null +++ b/Solr/Query/Expander.php @@ -0,0 +1,44 @@ + + */ +class Expander implements ExpanderInterface +{ + /** + * @var ExpansionInterface[] + */ + protected $expansions = []; + + /** + * {@inheritdoc} + */ + public function addExpansion(ExpansionInterface $expansion) + { + $this->expansions[] = $expansion; + return $this; + } + + /** + * {@inheritdoc} + */ + public function expand(AbstractQuery $query) + { + foreach ($this->expansions as $expansion) { + if (!$expansion->supportsClass($query)) { + continue; + } + + $expansion->expand($query); + } + + return $query; + } +} \ No newline at end of file diff --git a/Tests/Solr/Query/ExpanderTest.php b/Tests/Solr/Query/ExpanderTest.php new file mode 100644 index 0000000..5648e3a --- /dev/null +++ b/Tests/Solr/Query/ExpanderTest.php @@ -0,0 +1,62 @@ + + */ +class ExpanderTest extends \PHPUnit_Framework_TestCase +{ + /** + * @var Expander + */ + protected $expander; + + public function setup() + { + $this->expander = new Expander(); + } + + public function testExpand() + { + /** @var ExpansionInterface | \PHPUnit_Framework_MockObject_MockObject $expansion1 */ + $expansion1 = $this->getMock(ExpansionInterface::class); + + $expansion1 + ->expects($this->once()) + ->method('supportsClass') + ->willReturn(true) + ; + + $expansion1 + ->expects($this->once()) + ->method('expand') + ; + + /** @var ExpansionInterface | \PHPUnit_Framework_MockObject_MockObject $expansion2 */ + $expansion2 = $this->getMock(ExpansionInterface::class); + + $expansion2 + ->expects($this->once()) + ->method('supportsClass') + ->willReturn(false) + ; + + $expansion2 + ->expects($this->never()) + ->method('expand') + ; + + /** @var AbstractQuery | $query */ + $query = $this->getMock(AbstractQuery::class); + + $this->assertSame($this->expander, $this->expander->addExpansion($expansion1)); + $this->assertSame($this->expander, $this->expander->addExpansion($expansion2)); + $this->assertSame($query, $this->expander->expand($query)); + } +} From ce7820bda65fe9ca761b8cd3e2c2764957c915f9 Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Mon, 2 Jan 2017 13:37:54 +0100 Subject: [PATCH 16/48] Added compiler pass for registrering query expansions --- .../CompilerPass/ExpanderPass.php | 29 +++++++++++++++++++ IntegratedSolrBundle.php | 3 ++ 2 files changed, 32 insertions(+) create mode 100644 DependencyInjection/CompilerPass/ExpanderPass.php diff --git a/DependencyInjection/CompilerPass/ExpanderPass.php b/DependencyInjection/CompilerPass/ExpanderPass.php new file mode 100644 index 0000000..777327d --- /dev/null +++ b/DependencyInjection/CompilerPass/ExpanderPass.php @@ -0,0 +1,29 @@ + + */ +class ExpanderPass implements CompilerPassInterface +{ + /** + * {@inheritdoc} + */ + public function process(ContainerBuilder $container) + { + if (!$container->hasDefinition('integrated_solr.solr_query.expander')) { + return; + } + + $definition = $container->getDefinition('integrated_solr.solr_query.expander'); + + foreach ($container->findTaggedServiceIds('integrated_solr.expansion') as $service => $tags) { + $definition->addMethodCall('addExpansion', [new Reference($service)]); + } + } +} diff --git a/IntegratedSolrBundle.php b/IntegratedSolrBundle.php index c44e7d0..a2078c9 100644 --- a/IntegratedSolrBundle.php +++ b/IntegratedSolrBundle.php @@ -11,6 +11,7 @@ namespace Integrated\Bundle\SolrBundle; +use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\ExpanderPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterConfigFileProviderPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTaskHandlerPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTypePass; @@ -43,6 +44,8 @@ public function build(ContainerBuilder $container) 'integrated_solr.event_listener', 'integrated_solr.event_subscriber' )); + + $container->addCompilerPass(new ExpanderPass()); } /** From df08bf84bbd096aaf7c53855e1dabc33d653b908 Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Mon, 2 Jan 2017 15:01:02 +0100 Subject: [PATCH 17/48] Load the Solr xml --- DependencyInjection/IntegratedSolrExtension.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 5167731..72da427 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -45,6 +45,7 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('task.xml'); $loader->load('types.xml'); $loader->load('worker.xml'); + $loader->load('solr.xml'); if ($container->getParameter('kernel.debug')) { $loader->load('collector.xml'); From 6259b9f3bda30b1d8ec09478b60e5e5f7cbe53d4 Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Wed, 4 Jan 2017 12:41:51 +0100 Subject: [PATCH 18/48] Revert "Added Query expander" This reverts commit 827953058439c8ab91846684e576479cc630954a. --- Resources/config/solr.xml | 11 ------ Solr/Query/Expander.php | 44 ---------------------- Tests/Solr/Query/ExpanderTest.php | 62 ------------------------------- 3 files changed, 117 deletions(-) delete mode 100644 Resources/config/solr.xml delete mode 100644 Solr/Query/Expander.php delete mode 100644 Tests/Solr/Query/ExpanderTest.php diff --git a/Resources/config/solr.xml b/Resources/config/solr.xml deleted file mode 100644 index 62064c8..0000000 --- a/Resources/config/solr.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/Solr/Query/Expander.php b/Solr/Query/Expander.php deleted file mode 100644 index 5fcb5f7..0000000 --- a/Solr/Query/Expander.php +++ /dev/null @@ -1,44 +0,0 @@ - - */ -class Expander implements ExpanderInterface -{ - /** - * @var ExpansionInterface[] - */ - protected $expansions = []; - - /** - * {@inheritdoc} - */ - public function addExpansion(ExpansionInterface $expansion) - { - $this->expansions[] = $expansion; - return $this; - } - - /** - * {@inheritdoc} - */ - public function expand(AbstractQuery $query) - { - foreach ($this->expansions as $expansion) { - if (!$expansion->supportsClass($query)) { - continue; - } - - $expansion->expand($query); - } - - return $query; - } -} \ No newline at end of file diff --git a/Tests/Solr/Query/ExpanderTest.php b/Tests/Solr/Query/ExpanderTest.php deleted file mode 100644 index 5648e3a..0000000 --- a/Tests/Solr/Query/ExpanderTest.php +++ /dev/null @@ -1,62 +0,0 @@ - - */ -class ExpanderTest extends \PHPUnit_Framework_TestCase -{ - /** - * @var Expander - */ - protected $expander; - - public function setup() - { - $this->expander = new Expander(); - } - - public function testExpand() - { - /** @var ExpansionInterface | \PHPUnit_Framework_MockObject_MockObject $expansion1 */ - $expansion1 = $this->getMock(ExpansionInterface::class); - - $expansion1 - ->expects($this->once()) - ->method('supportsClass') - ->willReturn(true) - ; - - $expansion1 - ->expects($this->once()) - ->method('expand') - ; - - /** @var ExpansionInterface | \PHPUnit_Framework_MockObject_MockObject $expansion2 */ - $expansion2 = $this->getMock(ExpansionInterface::class); - - $expansion2 - ->expects($this->once()) - ->method('supportsClass') - ->willReturn(false) - ; - - $expansion2 - ->expects($this->never()) - ->method('expand') - ; - - /** @var AbstractQuery | $query */ - $query = $this->getMock(AbstractQuery::class); - - $this->assertSame($this->expander, $this->expander->addExpansion($expansion1)); - $this->assertSame($this->expander, $this->expander->addExpansion($expansion2)); - $this->assertSame($query, $this->expander->expand($query)); - } -} From 0363cde72017f5fe5ba62b7b4b3684a74d512738 Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Wed, 4 Jan 2017 12:42:53 +0100 Subject: [PATCH 19/48] Revert "Added compiler pass for registrering query expansions" This reverts commit ce7820bda65fe9ca761b8cd3e2c2764957c915f9. --- .../CompilerPass/ExpanderPass.php | 29 ------------------- IntegratedSolrBundle.php | 3 -- 2 files changed, 32 deletions(-) delete mode 100644 DependencyInjection/CompilerPass/ExpanderPass.php diff --git a/DependencyInjection/CompilerPass/ExpanderPass.php b/DependencyInjection/CompilerPass/ExpanderPass.php deleted file mode 100644 index 777327d..0000000 --- a/DependencyInjection/CompilerPass/ExpanderPass.php +++ /dev/null @@ -1,29 +0,0 @@ - - */ -class ExpanderPass implements CompilerPassInterface -{ - /** - * {@inheritdoc} - */ - public function process(ContainerBuilder $container) - { - if (!$container->hasDefinition('integrated_solr.solr_query.expander')) { - return; - } - - $definition = $container->getDefinition('integrated_solr.solr_query.expander'); - - foreach ($container->findTaggedServiceIds('integrated_solr.expansion') as $service => $tags) { - $definition->addMethodCall('addExpansion', [new Reference($service)]); - } - } -} diff --git a/IntegratedSolrBundle.php b/IntegratedSolrBundle.php index a2078c9..c44e7d0 100644 --- a/IntegratedSolrBundle.php +++ b/IntegratedSolrBundle.php @@ -11,7 +11,6 @@ namespace Integrated\Bundle\SolrBundle; -use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\ExpanderPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterConfigFileProviderPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTaskHandlerPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTypePass; @@ -44,8 +43,6 @@ public function build(ContainerBuilder $container) 'integrated_solr.event_listener', 'integrated_solr.event_subscriber' )); - - $container->addCompilerPass(new ExpanderPass()); } /** From 585fc0ae4044e421db47fcd0bd03231d624ed3ac Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Wed, 4 Jan 2017 12:43:10 +0100 Subject: [PATCH 20/48] Revert "Load the Solr xml" This reverts commit df08bf84bbd096aaf7c53855e1dabc33d653b908. --- DependencyInjection/IntegratedSolrExtension.php | 1 - 1 file changed, 1 deletion(-) diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 72da427..5167731 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -45,7 +45,6 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('task.xml'); $loader->load('types.xml'); $loader->load('worker.xml'); - $loader->load('solr.xml'); if ($container->getParameter('kernel.debug')) { $loader->load('collector.xml'); From df4fcaaad22bc1402c18c044bf93a3cbb1d801b9 Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Wed, 4 Jan 2017 12:43:48 +0100 Subject: [PATCH 21/48] Inject the event dispatcher into the solr client --- DependencyInjection/IntegratedSolrExtension.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 5167731..e8312fc 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -67,7 +67,13 @@ public function load(array $configs, ContainerBuilder $container) $container->setDefinition( 'solarium.client', - new Definition('%integrated_solr.solarium.client.class%', array(array('endpoint' => $endpoints))) + new Definition( + '%integrated_solr.solarium.client.class%', + [ + ['endpoint' => $endpoints], + new Reference('integrated_solr.event.dispatcher') + ] + ) ); if ($container->getParameter('kernel.debug')) { From b4690052ad818842f1139a6fc6518c992e2ba420 Mon Sep 17 00:00:00 2001 From: Ger Jan Date: Wed, 14 Dec 2016 15:51:51 +0100 Subject: [PATCH 22/48] [INTEGRATED-947] Don't hydrate to improve indexer performance --- Command/IndexerQueueCommand.php | 35 +++++++++++---------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/Command/IndexerQueueCommand.php b/Command/IndexerQueueCommand.php index 8bc8f7a..35d1506 100644 --- a/Command/IndexerQueueCommand.php +++ b/Command/IndexerQueueCommand.php @@ -14,11 +14,12 @@ use DateTime; use DateTimeZone; -use Integrated\Common\Content\ContentInterface; use Integrated\Common\ContentType\ResolverInterface; use Integrated\Common\Solr\Indexer\Job; use Integrated\Common\Queue\QueueInterface; +use Integrated\Bundle\ContentBundle\Document\Content\Content; + use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; use Symfony\Component\Console\Helper\ProgressHelper; @@ -175,32 +176,21 @@ protected function executeDelete(InputInterface $input, OutputInterface $output) */ protected function executeIndex(InputInterface $input, OutputInterface $output) { - $result = null; + // Don't hydrate for performance reasons + $builder = $this->getDocumentManager()->createQueryBuilder(Content::class); + $builder->select('id', 'contentType', 'class')->hydrate(false); if ($input->getOption('full')) { - - //use createQueryBuilder for performance reasons - $qb = $this->getDocumentManager()->createQueryBuilder( - 'Integrated\Bundle\ContentBundle\Document\Content\Content' - ) - ->select('id', 'contentType', 'class'); - $result = $qb->getQuery()->execute(); + $result = $builder->getQuery()->execute(); // The entire site is going to be reindex so everything that is now in the queue // will be redone so just clear it so content is not double indexed. $this->getQueue()->clear(); } else { - $criteria['$or'] = []; + $builder->field('contentType')->in($input->getArgument('id')); - foreach ($input->getArgument('id') as $id) { - $criteria['$or'][] = ['contentType' => $id]; - } - - $result = $this->getDocumentManager() - ->getUnitOfWork() - ->getDocumentPersister('Integrated\Bundle\ContentBundle\Document\Content\Content') - ->loadAll($criteria); + $result = $builder->getQuery()->execute(); } if ($count = $result->count()) { @@ -251,17 +241,14 @@ protected function doIndex(Cursor $cursor, ProgressHelper $progress) $count = 0; $manager = $this->getDocumentManager(); - /** @var ContentInterface $document */ - foreach ($cursor as $document) { $progress->advance(); $job = new Job('ADD'); - $job->setOption('document.id', $document->getContentType() . '-' . $document->getId()); - - $job->setOption('document.data', $this->getSerializer()->serialize($document, 'json')); - $job->setOption('document.class', get_class($document)); + $job->setOption('document.id', $document['contentType'] . '-' . $document['_id']); + $job->setOption('document.data', json_encode(['id' => $document['_id']])); + $job->setOption('document.class', $document['class']); $job->setOption('document.format', 'json'); $queue->push($job); From fe2c58bfb31fb09e9e36f0659258da855b54f8ef Mon Sep 17 00:00:00 2001 From: Ger Jan Date: Wed, 14 Dec 2016 15:56:24 +0100 Subject: [PATCH 23/48] [INTEGRATED-947] ContentType can be optional --- Command/IndexerQueueCommand.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Command/IndexerQueueCommand.php b/Command/IndexerQueueCommand.php index 35d1506..92952ec 100644 --- a/Command/IndexerQueueCommand.php +++ b/Command/IndexerQueueCommand.php @@ -246,7 +246,10 @@ protected function doIndex(Cursor $cursor, ProgressHelper $progress) $job = new Job('ADD'); - $job->setOption('document.id', $document['contentType'] . '-' . $document['_id']); + $contentType = isset($document['contentType']) ? $document['contentType'] : ''; + + $job->setOption('document.id', $contentType . '-' . $document['_id']); + $job->setOption('document.data', json_encode(['id' => $document['_id']])); $job->setOption('document.class', $document['class']); $job->setOption('document.format', 'json'); From 0baa5aa614fbbdb2fa16f21104911fa05c6fd245 Mon Sep 17 00:00:00 2001 From: Jeroen van Leeuwen Date: Fri, 6 Jan 2017 10:25:44 +0100 Subject: [PATCH 24/48] Removed the compiler because the event dispatcher is injected into the client --- .../SolariumEventTagCompilerPass.php | 33 ------------------- IntegratedSolrBundle.php | 2 -- 2 files changed, 35 deletions(-) delete mode 100644 DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php diff --git a/DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php b/DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php deleted file mode 100644 index 1d03162..0000000 --- a/DependencyInjection/CompilerPass/SolariumEventTagCompilerPass.php +++ /dev/null @@ -1,33 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass; - -use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; -use Symfony\Component\DependencyInjection\ContainerBuilder; -use Symfony\Component\DependencyInjection\Reference; - -/** - * @author Johnny Borg - */ -class SolariumEventTagCompilerPass implements CompilerPassInterface -{ - /** - * {@inheritdoc} - */ - public function process(ContainerBuilder $container) - { - $definition = $container->getDefinition('solarium.client'); - foreach ($container->findTaggedServiceIds('integrated_solr.event_subscriber') as $service => $tags) { - $definition ->addMethodCall('getEventDispatcher()->addSubscriber', [new Reference($service)]); - } - } -} diff --git a/IntegratedSolrBundle.php b/IntegratedSolrBundle.php index c44e7d0..cb2ebb6 100644 --- a/IntegratedSolrBundle.php +++ b/IntegratedSolrBundle.php @@ -14,7 +14,6 @@ use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterConfigFileProviderPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTaskHandlerPass; use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\RegisterTypePass; -use Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass\SolariumEventTagCompilerPass; use Integrated\Bundle\SolrBundle\DependencyInjection\IntegratedSolrExtension; use Symfony\Component\EventDispatcher\DependencyInjection\RegisterListenersPass; @@ -33,7 +32,6 @@ public function build(ContainerBuilder $container) { parent::build($container); - $container->addCompilerPass(new SolariumEventTagCompilerPass()); $container->addCompilerPass(new RegisterConfigFileProviderPass()); $container->addCompilerPass(new RegisterTypePass()); $container->addCompilerPass(new RegisterTaskHandlerPass()); From 5e190836faca6c7a6a5f1ecdf87c6c89558a6bdc Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Thu, 15 Dec 2016 17:41:18 +0100 Subject: [PATCH 25/48] the process based indexer # Conflicts: # Command/IndexerRunCommand.php # DependencyInjection/IntegratedSolrExtension.php --- Command/IndexerRunCommand.php | 158 ++++++++++++++---- .../IntegratedSolrExtension.php | 1 + Process/ArgumentProcess.php | 80 +++++++++ Process/Exception/FormatException.php | 26 +++ Process/Exception/LogicException.php | 34 ++++ Process/ProcessPoolGenerator.php | 71 ++++++++ Resources/config/command.xml | 25 +++ 7 files changed, 366 insertions(+), 29 deletions(-) create mode 100644 Process/ArgumentProcess.php create mode 100644 Process/Exception/FormatException.php create mode 100644 Process/Exception/LogicException.php create mode 100644 Process/ProcessPoolGenerator.php create mode 100644 Resources/config/command.xml diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index cc21eec..bfaf2d0 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -11,22 +11,60 @@ namespace Integrated\Bundle\SolrBundle\Command; +use Exception; + +use Integrated\Bundle\SolrBundle\Process\ArgumentProcess; +use Integrated\Bundle\SolrBundle\Process\ProcessPoolGenerator; + +use Integrated\Common\Queue\Provider\DBAL\QueueProvider; +use Integrated\Common\Queue\Queue; +use Integrated\Common\Solr\Indexer\Indexer; +use Integrated\Common\Solr\Indexer\IndexerInterface; + use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; + +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Process\Process; use Symfony\Component\Filesystem\LockHandler; -use Exception; - -use Integrated\Common\Solr\Indexer\IndexerInterface; - /** * @author Jan Sanne Mulder */ -class IndexerRunCommand extends ContainerAwareCommand +class IndexerRunCommand extends Command { + /** + * @var Indexer + */ + protected $indexer; + + /** + * @var QueueProvider + */ + protected $queueProvider; + + /** + * @var string + */ + protected $workingDirectory; + + /** + * @param Indexer $indexer + * @param Queue $solrQueue + * @param string $workingDirectory + */ + public function __construct(Indexer $indexer, QueueProvider $queueProvider, $workingDirectory) + { + parent::__construct(); + + $this->indexer = $indexer; + $this->queueProvider = $queueProvider; + $this->workingDirectory = $workingDirectory; + } + /** * @see Command */ @@ -48,6 +86,18 @@ protected function configure() 'Time in milliseconds to wait between runs (in combination with --full or --daemon)', 0 ) + ->addArgument( + 'processes', + InputArgument::OPTIONAL, + 'Creates a number of proccess that run the queue', + 0 + ) + ->addOption( + 'blocking', + 'b', + InputOption::VALUE_NONE, + 0 + ) ->setDescription('Execute a sol indexer run') ->setHelp(' The %command.name% command starts a indexer run. @@ -58,38 +108,44 @@ protected function configure() } /** - * @see Command::execute() + * @param InputInterface $input + * @param OutputInterface $output + * @return int */ protected function execute(InputInterface $input, OutputInterface $output) { - if ($input->getOption('full') || $input->getOption('daemon')) { - return $this->runExternal($input, $output); + if ($argument = $input->getArgument('processes')) { + return $this->runProcess(new ArgumentProcess($argument), $input, $output); + } else if ($input->getOption('full') || $input->getOption('daemon')) { + return $this->runExternal($input); } - return $this->runInternal($input, $output); + return $this->runInternal(self::class, $output); } /** - * @param InputInterface $input + * @param string $lock * @param OutputInterface $output * @return int */ - private function runInternal(InputInterface $input, OutputInterface $output) + private function runInternal($lock, OutputInterface $output) { try { - $lock = new LockHandler('Integrated\Bundle\SolrBundle\Command\IndexerRunCommand'); - $attemps = 0; + $lock = new LockHandler($lock); + $attempts = 0; while (!$lock->lock()) { - //retry for almost a minute, otherwise don't throw an error (after all another indexer is running) - if ($attemps++ >= 10) { + // Retry for almost a minute, otherwise don't throw an error (after all another indexer is running) + if ($attempts++ >= 10) { return 0; } sleep(5); } - /** @var IndexerInterface $indexer */ - $indexer = $this->getContainer()->get('integrated_solr.indexer'); - $indexer->execute(); + if ($output->isDebug() && method_exists($this->indexer, 'setDebug')) { + $this->indexer->setDebug(); + } + + $this->indexer->execute(); } catch (Exception $e) { $output->writeln("Aborting: " . $e->getMessage()); @@ -102,19 +158,16 @@ private function runInternal(InputInterface $input, OutputInterface $output) /** * @param InputInterface $input - * @param OutputInterface $output * @return int */ - private function runExternal(InputInterface $input, OutputInterface $output) + private function runExternal(InputInterface $input) { - $wait = (int)$input->getOption('wait'); + $wait = (int) $input->getOption('wait'); $wait = $wait * 1000; // convert from milli to micro while (true) { - $process = new Process( - 'php app/console solr:indexer:run -e ' . $input->getOption('env'), - $this->getRootDir() - ); + // Run a external process + $process = new Process('php app/console solr:indexer:run', $this->workingDirectory); $process->setTimeout(0); $process->run(function ($type, $buffer) use ($output) { @@ -126,7 +179,7 @@ private function runExternal(InputInterface $input, OutputInterface $output) } if (!$input->getOption('daemon')) { - if (!$this->getContainer()->get('integrated_solr.indexer')->getQueue()->count()) { + if ($this->indexer->getQueue()->count()) { break; } } @@ -138,10 +191,57 @@ private function runExternal(InputInterface $input, OutputInterface $output) } /** - * @return string + * @param ArgumentProcess $argumentProcess + * @param InputInterface $input + * @param OutputInterface $output + * @return int */ - protected function getRootDir() + private function runProcess(ArgumentProcess $argument, InputInterface $input, OutputInterface $output) { - return realpath($this->getContainer()->get('kernel')->getRootDir() . '/..'); + if ($argument->isParentProcess()) { + // Create pool generator to generate the processes + $generator = new ProcessPoolGenerator($input); + $pool = $generator->getProcessesPool($argument, $this->workingDirectory); + + // Start them accordingly + foreach ($pool as $i => $process) { + // Run it + $process->start(); + + // Tell somebody + $output->writeln(sprintf('Started process %d with pid %d to run the queue.', ($i+1), $process->getPid())); + } + + if ($input->getOption('blocking')) { + $output->writeln('Running in blocking mode, waiting until all started process are done'); + + // While the pool contains processes we're running + while ($pool->count()) { + foreach ($pool as $i => $process) { + if (!$process->isRunning()) { + $output->writeln(sprintf('Process %d finnished.', ($i+1))); + + // This one is important + $pool->removeElement($process); + } + } + + // Don't create a cpu load, check periodically + sleep(1); + } + } + + // Good to go + return 0; + } + + // Set the modulo to run over the dataset with x processes + $this->queueProvider->setOption('where', sprintf('(id %% %d) = %d', $argument->getProcessMax(), $argument->getProcessNumber())); + + // Remove the limit so we'll keep on johnny walk'n + $this->indexer->setOption('queue.size', -1); + + // Seems to be a sub-process, ran it with a the number appended to the class + return $this->runInternal(sprintf('%s:%d', self::class, $argument->getProcessNumber()), $output); } } diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index e8312fc..1b004ca 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -39,6 +39,7 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('converter.xml'); $loader->load('event.xml'); + $loader->load('command.xml'); $loader->load('indexer.xml'); $loader->load('queue.xml'); $loader->load('solarium.xml'); diff --git a/Process/ArgumentProcess.php b/Process/ArgumentProcess.php new file mode 100644 index 0000000..1c69fb0 --- /dev/null +++ b/Process/ArgumentProcess.php @@ -0,0 +1,80 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Process; + +use Integrated\Bundle\SolrBundle\Process\Exception\FormatException; +use Integrated\Bundle\SolrBundle\Process\Exception\LogicException; + +/** + * @author Johnny Borg + */ +class ArgumentProcess +{ + /** + * @const string + */ + const FORMAT = '/^(\d):(\d)$/'; + + /** + * @var string + */ + protected $argument; + + /** + * @param string $argument + */ + public function __construct($argument) + { + $this->argument = $argument; + } + + /** + * @return bool + */ + public function isParentProcess() + { + return (false === strpos($this->argument, ':')); + } + + /** + * @return int + */ + public function getProcessNumber() + { + if (!$this->isParentProcess()) { + if (preg_match(self::FORMAT, $this->argument, $matches)) { + return $matches[1]; + } + + throw FormatException::noRegexMatch(); + } + + throw LogicException::invalidMethodCall(); + } + + /** + * @return int + * @throws FormatException + */ + public function getProcessMax() + { + if (!$this->isParentProcess()) { + if (preg_match(self::FORMAT, $this->argument, $matches)) { + return $matches[2]; + } + + throw FormatException::noRegexMatch(); + } + + return (int) $this->argument; + } +} diff --git a/Process/Exception/FormatException.php b/Process/Exception/FormatException.php new file mode 100644 index 0000000..4ce97fe --- /dev/null +++ b/Process/Exception/FormatException.php @@ -0,0 +1,26 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Process\Exception; + +/** + * @author Johnny Borg + */ +class FormatException extends \Exception +{ + /** + * @return static + */ + public static function noRegexMatch() + { + return new static('Format does not required pattern'); + } +} diff --git a/Process/Exception/LogicException.php b/Process/Exception/LogicException.php new file mode 100644 index 0000000..a598245 --- /dev/null +++ b/Process/Exception/LogicException.php @@ -0,0 +1,34 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Process\Exception; + +/** + * @author Johnny Borg + */ +class LogicException extends \Exception +{ + /** + * @return static + */ + public static function invalidMethodCall() + { + return new static('This method should not be called in this context'); + } + + /** + * @return static + */ + public static function noProcessesGenerated() + { + return new static('No processes could be generated with given input'); + } +} diff --git a/Process/ProcessPoolGenerator.php b/Process/ProcessPoolGenerator.php new file mode 100644 index 0000000..1a78faf --- /dev/null +++ b/Process/ProcessPoolGenerator.php @@ -0,0 +1,71 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\Process; + +use Doctrine\Common\Collections\ArrayCollection; + +use Integrated\Bundle\SolrBundle\Process\Exception\LogicException; + +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Process\Process; + +/** + * @author Johnny Borg + */ +class ProcessPoolGenerator +{ + /** + * @const + */ + const COMMAND = 'php app/console %s %s %d:%d'; + + /** + * @var InputInterface + */ + private $input; + + /** + * @param ArgumentProcess $argumentProcess + */ + public function __construct(InputInterface $input) + { + $this->input = $input; + } + + /** + * @param ArgumentProcess $argumentProcess + * @return ArrayCollection|Process[] + */ + public function getProcessesPool(ArgumentProcess $argumentProcess, $workingDirectory) + { + $result = new ArrayCollection(); + + for ($i = 0; $i < $argumentProcess->getProcessMax(); $i++) { + $result[] = new Process( + sprintf( + self::COMMAND, + $this->input->getFirstArgument(), + $this->input->getParameterOption('command'), + $i, + $argumentProcess->getProcessMax() + ), + $workingDirectory + ); + } + + if ($result->count()) { + return $result; + } + + throw LogicException::noProcessesGenerated(); + } +} diff --git a/Resources/config/command.xml b/Resources/config/command.xml new file mode 100644 index 0000000..f9ed9c8 --- /dev/null +++ b/Resources/config/command.xml @@ -0,0 +1,25 @@ + + + + + + + Integrated\Bundle\SolrBundle\Command\IndexerRunCommand + + + + + + + + + %kernel.root_dir%/.. + + + + + + + From e88ecefd31a3d57c9ff68587404f4286f6e771d1 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Fri, 16 Dec 2016 15:02:21 +0100 Subject: [PATCH 26/48] Added environment to the command. --- Command/IndexerRunCommand.php | 19 ++++++++++++++----- Process/ProcessPoolGenerator.php | 19 +++++++++++++++---- Resources/config/command.xml | 1 + 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index bfaf2d0..ea501d6 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -28,6 +28,8 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\HttpKernel\Kernel; +use Symfony\Component\HttpKernel\KernelInterface; use Symfony\Component\Process\Process; use Symfony\Component\Filesystem\LockHandler; @@ -51,18 +53,25 @@ class IndexerRunCommand extends Command */ protected $workingDirectory; + /** + * @var KernelInterface + */ + private $kernel; + /** * @param Indexer $indexer - * @param Queue $solrQueue + * @param QueueProvider $queueProvider + * @param KernelInterface $kernel * @param string $workingDirectory */ - public function __construct(Indexer $indexer, QueueProvider $queueProvider, $workingDirectory) + public function __construct(Indexer $indexer, QueueProvider $queueProvider, KernelInterface $kernel, $workingDirectory) { parent::__construct(); $this->indexer = $indexer; $this->queueProvider = $queueProvider; $this->workingDirectory = $workingDirectory; + $this->kernel = $kernel; } /** @@ -167,7 +176,7 @@ private function runExternal(InputInterface $input) while (true) { // Run a external process - $process = new Process('php app/console solr:indexer:run', $this->workingDirectory); + $process = new Process('php app/console solr:indexer:run -e ', $this->workingDirectory); $process->setTimeout(0); $process->run(function ($type, $buffer) use ($output) { @@ -191,7 +200,7 @@ private function runExternal(InputInterface $input) } /** - * @param ArgumentProcess $argumentProcess + * @param ArgumentProcess $argument * @param InputInterface $input * @param OutputInterface $output * @return int @@ -200,7 +209,7 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou { if ($argument->isParentProcess()) { // Create pool generator to generate the processes - $generator = new ProcessPoolGenerator($input); + $generator = new ProcessPoolGenerator($input, $this->kernel); $pool = $generator->getProcessesPool($argument, $this->workingDirectory); // Start them accordingly diff --git a/Process/ProcessPoolGenerator.php b/Process/ProcessPoolGenerator.php index 1a78faf..3d0e293 100644 --- a/Process/ProcessPoolGenerator.php +++ b/Process/ProcessPoolGenerator.php @@ -16,6 +16,7 @@ use Integrated\Bundle\SolrBundle\Process\Exception\LogicException; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\HttpKernel\Kernel; use Symfony\Component\Process\Process; /** @@ -26,7 +27,7 @@ class ProcessPoolGenerator /** * @const */ - const COMMAND = 'php app/console %s %s %d:%d'; + const COMMAND = 'php app/console %s %s %d:%d -e %s'; /** * @var InputInterface @@ -34,16 +35,25 @@ class ProcessPoolGenerator private $input; /** - * @param ArgumentProcess $argumentProcess + * @var Kernel + */ + private $kernel; + + /** + * @param InputInterface $input + * @param Kernel $kernel */ - public function __construct(InputInterface $input) + public function __construct(InputInterface $input, Kernel $kernel) { $this->input = $input; + $this->kernel = $kernel; } /** * @param ArgumentProcess $argumentProcess + * @param string $workingDirectory * @return ArrayCollection|Process[] + * @throws LogicException */ public function getProcessesPool(ArgumentProcess $argumentProcess, $workingDirectory) { @@ -56,7 +66,8 @@ public function getProcessesPool(ArgumentProcess $argumentProcess, $workingDirec $this->input->getFirstArgument(), $this->input->getParameterOption('command'), $i, - $argumentProcess->getProcessMax() + $argumentProcess->getProcessMax(), + $this->kernel->getEnvironment() ), $workingDirectory ); diff --git a/Resources/config/command.xml b/Resources/config/command.xml index f9ed9c8..b200099 100644 --- a/Resources/config/command.xml +++ b/Resources/config/command.xml @@ -15,6 +15,7 @@ + %kernel.root_dir%/.. From 5b59798bf8216435921afd61a020334daeac228c Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Fri, 16 Dec 2016 15:32:26 +0100 Subject: [PATCH 27/48] add environment --- Command/IndexerRunCommand.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index ea501d6..732381e 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -176,7 +176,10 @@ private function runExternal(InputInterface $input) while (true) { // Run a external process - $process = new Process('php app/console solr:indexer:run -e ', $this->workingDirectory); + $process = new Process( + sprintf('php app/console solr:indexer:run -e %s', $this->kernel->getEnvironment()), + $this->workingDirectory + ); $process->setTimeout(0); $process->run(function ($type, $buffer) use ($output) { From dc856964bced53563add99d90e6fcbbdae1a90c8 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Fri, 16 Dec 2016 15:41:34 +0100 Subject: [PATCH 28/48] Typo --- Command/IndexerRunCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index 732381e..c97b038 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -231,7 +231,7 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou while ($pool->count()) { foreach ($pool as $i => $process) { if (!$process->isRunning()) { - $output->writeln(sprintf('Process %d finnished.', ($i+1))); + $output->writeln(sprintf('Process %d finished.', ($i+1))); // This one is important $pool->removeElement($process); From 9f64a620c9bae4ac1d05ef461e000ae45374c8b7 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Mon, 19 Dec 2016 09:40:18 +0100 Subject: [PATCH 29/48] removed dots --- Command/IndexerRunCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index c97b038..4a39732 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -221,7 +221,7 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou $process->start(); // Tell somebody - $output->writeln(sprintf('Started process %d with pid %d to run the queue.', ($i+1), $process->getPid())); + $output->writeln(sprintf('Started process %d with pid %d to run the queue', ($i+1), $process->getPid())); } if ($input->getOption('blocking')) { @@ -231,7 +231,7 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou while ($pool->count()) { foreach ($pool as $i => $process) { if (!$process->isRunning()) { - $output->writeln(sprintf('Process %d finished.', ($i+1))); + $output->writeln(sprintf('Process %d finished', ($i+1))); // This one is important $pool->removeElement($process); From 2808fb87e1365c89d54621856e502e1062622efd Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Mon, 19 Dec 2016 12:30:27 +0100 Subject: [PATCH 30/48] Added memory saving strategy. # Conflicts: # DependencyInjection/IntegratedSolrExtension.php --- Command/IndexerRunCommand.php | 20 ++++-- .../IntegratedSolrExtension.php | 1 + .../DoctrineClearEventSubscriber.php | 65 +++++++++++++++++++ Resources/config/command.xml | 3 + Resources/config/subscriber.xml | 23 +++++++ 5 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 EventSubscriber/DoctrineClearEventSubscriber.php create mode 100644 Resources/config/subscriber.xml diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index 4a39732..c0052ea 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -13,6 +13,8 @@ use Exception; +use Integrated\Bundle\SolrBundle\EventSubscriber\DoctrineClearEventSubscriber; +use Integrated\Bundle\SolrBundle\EventSubscriber\MemoryEventSubscriber; use Integrated\Bundle\SolrBundle\Process\ArgumentProcess; use Integrated\Bundle\SolrBundle\Process\ProcessPoolGenerator; @@ -53,24 +55,31 @@ class IndexerRunCommand extends Command */ protected $workingDirectory; + /** + * @var DoctrineClearEventSubscriber + */ + protected $clearEventSubscriber; + /** * @var KernelInterface */ - private $kernel; + protected $kernel; /** * @param Indexer $indexer * @param QueueProvider $queueProvider * @param KernelInterface $kernel + * @param DoctrineClearEventSubscriber $clearEventSubscriber * @param string $workingDirectory */ - public function __construct(Indexer $indexer, QueueProvider $queueProvider, KernelInterface $kernel, $workingDirectory) + public function __construct(Indexer $indexer, QueueProvider $queueProvider, DoctrineClearEventSubscriber $clearEventSubscriber, KernelInterface $kernel, $workingDirectory) { parent::__construct(); $this->indexer = $indexer; $this->queueProvider = $queueProvider; $this->workingDirectory = $workingDirectory; + $this->clearEventSubscriber = $clearEventSubscriber; $this->kernel = $kernel; } @@ -225,7 +234,7 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou } if ($input->getOption('blocking')) { - $output->writeln('Running in blocking mode, waiting until all started process are done'); + $output->writeln('Running in blocking mode, waiting until all started processes are done'); // While the pool contains processes we're running while ($pool->count()) { @@ -247,9 +256,12 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou return 0; } - // Set the modulo to run over the dataset with x processes + // Set the modulo to run over the data set with x processes, creating a unique list per thread $this->queueProvider->setOption('where', sprintf('(id %% %d) = %d', $argument->getProcessMax(), $argument->getProcessNumber())); + // Add the clear event listener only for the thread + $this->indexer->getEventDispatcher()->addSubscriber($this->clearEventSubscriber); + // Remove the limit so we'll keep on johnny walk'n $this->indexer->setOption('queue.size', -1); diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 1b004ca..82d4833 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -44,6 +44,7 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('queue.xml'); $loader->load('solarium.xml'); $loader->load('task.xml'); + $loader->load('subscriber.xml'); $loader->load('types.xml'); $loader->load('worker.xml'); diff --git a/EventSubscriber/DoctrineClearEventSubscriber.php b/EventSubscriber/DoctrineClearEventSubscriber.php new file mode 100644 index 0000000..59dece0 --- /dev/null +++ b/EventSubscriber/DoctrineClearEventSubscriber.php @@ -0,0 +1,65 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Integrated\Bundle\SolrBundle\EventSubscriber; + +use Doctrine\ODM\MongoDB\DocumentManager; +use Doctrine\ORM\EntityManager; + +use Integrated\Common\Solr\Indexer\Event\MessageEvent; +use Integrated\Common\Solr\Indexer\Events; + +use Symfony\Component\EventDispatcher\EventSubscriberInterface; + +/** + * @author Johnny Borg + */ +class DoctrineClearEventSubscriber implements EventSubscriberInterface +{ + /** + * @var DocumentManager + */ + private $documentManager; + + /** + * @var EntityManager + */ + private $entityManager; + + /** + * @param DocumentManager $documentManager + * @param EntityManager $entityManager + */ + public function __construct(DocumentManager $documentManager, EntityManager $entityManager) + { + $this->documentManager = $documentManager; + $this->entityManager = $entityManager; + } + + /** + * @return array + */ + public static function getSubscribedEvents() + { + return [ + Events::PROCESSED => 'processedEvent' + ]; + } + + /** + * @param MessageEvent $messageEvent + */ + public function processedEvent(MessageEvent $messageEvent) + { + $this->documentManager->clear(); + $this->entityManager->clear(); + } +} diff --git a/Resources/config/command.xml b/Resources/config/command.xml index b200099..bcfc84d 100644 --- a/Resources/config/command.xml +++ b/Resources/config/command.xml @@ -13,12 +13,15 @@ + + %kernel.root_dir%/.. + diff --git a/Resources/config/subscriber.xml b/Resources/config/subscriber.xml new file mode 100644 index 0000000..38e307f --- /dev/null +++ b/Resources/config/subscriber.xml @@ -0,0 +1,23 @@ + + + + + + + Integrated\Bundle\SolrBundle\EventSubscriber\DoctrineClearEventSubscriber + + + + + + + + + + + + + + \ No newline at end of file From f46a3a1a847571fe50bb4f382d94bfe235a85265 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Mon, 19 Dec 2016 16:01:09 +0100 Subject: [PATCH 31/48] Fix the number of starting threads. --- Process/ProcessPoolGenerator.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Process/ProcessPoolGenerator.php b/Process/ProcessPoolGenerator.php index 3d0e293..b2ee43b 100644 --- a/Process/ProcessPoolGenerator.php +++ b/Process/ProcessPoolGenerator.php @@ -59,7 +59,7 @@ public function getProcessesPool(ArgumentProcess $argumentProcess, $workingDirec { $result = new ArrayCollection(); - for ($i = 0; $i < $argumentProcess->getProcessMax(); $i++) { + for ($i = 0; $i <= $argumentProcess->getProcessMax(); $i++) { $result[] = new Process( sprintf( self::COMMAND, From db15111f4be20674c2215d42e5a9809414e8c673 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Tue, 20 Dec 2016 12:43:25 +0100 Subject: [PATCH 32/48] Should output when running in blocked mode. --- Command/IndexerRunCommand.php | 4 ++++ Process/ProcessPoolGenerator.php | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index c0052ea..f0ae52e 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -240,6 +240,10 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou while ($pool->count()) { foreach ($pool as $i => $process) { if (!$process->isRunning()) { + if ($process->getIncrementalOutput()) { + $output->writeln(sprintf('Prcocess %d: %s', $i, $process->getIncrementalOutput())); + } + $output->writeln(sprintf('Process %d finished', ($i+1))); // This one is important diff --git a/Process/ProcessPoolGenerator.php b/Process/ProcessPoolGenerator.php index b2ee43b..3d0e293 100644 --- a/Process/ProcessPoolGenerator.php +++ b/Process/ProcessPoolGenerator.php @@ -59,7 +59,7 @@ public function getProcessesPool(ArgumentProcess $argumentProcess, $workingDirec { $result = new ArrayCollection(); - for ($i = 0; $i <= $argumentProcess->getProcessMax(); $i++) { + for ($i = 0; $i < $argumentProcess->getProcessMax(); $i++) { $result[] = new Process( sprintf( self::COMMAND, From df528096597acf93c0f413030f82119edde9272a Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Tue, 20 Dec 2016 14:10:45 +0100 Subject: [PATCH 33/48] This should keep the group processes running a bit longer. --- Command/IndexerRunCommand.php | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index f0ae52e..d98b115 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -255,21 +255,26 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou sleep(1); } } + } else { + // Set the modulo to run over the data set with x processes, creating a unique list per thread + $this->queueProvider->setOption('where', sprintf('(id %% %d) = %d', $argument->getProcessMax(), $argument->getProcessNumber())); - // Good to go - return 0; - } + // Add the clear event listener only for the thread + $this->indexer->getEventDispatcher()->addSubscriber($this->clearEventSubscriber); - // Set the modulo to run over the data set with x processes, creating a unique list per thread - $this->queueProvider->setOption('where', sprintf('(id %% %d) = %d', $argument->getProcessMax(), $argument->getProcessNumber())); + // Remove the limit so we'll keep on johnny walk'n + //$this->indexer->setOption('queue.size', -1); - // Add the clear event listener only for the thread - $this->indexer->getEventDispatcher()->addSubscriber($this->clearEventSubscriber); + // Seems to be a sub-process, ran it with a the number appended to the class + while ($this->indexer->getQueue()) { + $this->runInternal(sprintf('%s:%d', self::class, $argument->getProcessNumber()), $output); - // Remove the limit so we'll keep on johnny walk'n - $this->indexer->setOption('queue.size', -1); + // Give them cores some relaxation + usleep(5000); + } + } - // Seems to be a sub-process, ran it with a the number appended to the class - return $this->runInternal(sprintf('%s:%d', self::class, $argument->getProcessNumber()), $output); + // Good to go + return 0; } } From bb10dd3872053a9494f3bd55ae32e259538465ea Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Wed, 21 Dec 2016 10:00:16 +0100 Subject: [PATCH 34/48] Keep running until the queue is really empty and pass along any output when running in blocked mode. --- Command/IndexerQueueCommand.php | 2 +- Command/IndexerRunCommand.php | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Command/IndexerQueueCommand.php b/Command/IndexerQueueCommand.php index 92952ec..5627176 100644 --- a/Command/IndexerQueueCommand.php +++ b/Command/IndexerQueueCommand.php @@ -186,7 +186,7 @@ protected function executeIndex(InputInterface $input, OutputInterface $output) // The entire site is going to be reindex so everything that is now in the queue // will be redone so just clear it so content is not double indexed. - $this->getQueue()->clear(); + //$this->getQueue()->clear(); } else { $builder->field('contentType')->in($input->getArgument('id')); diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index d98b115..8ae638e 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -239,11 +239,17 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou // While the pool contains processes we're running while ($pool->count()) { foreach ($pool as $i => $process) { - if (!$process->isRunning()) { - if ($process->getIncrementalOutput()) { - $output->writeln(sprintf('Prcocess %d: %s', $i, $process->getIncrementalOutput())); - } + // Read stout for anything to pass thru + if ($processOutput = $process->getIncrementalOutput()) { + $output->writeln(sprintf('Prcocess %d: %s', $i, $processOutput)); + } + // Read sterr for anything to pass thru + if ($processOutput = $process->getIncrementalErrorOutput()) { + $output->writeln(sprintf('Prcocess %d: %s', $i, $processOutput)); + } + if (!$process->isRunning()) { + // Tell the user $output->writeln(sprintf('Process %d finished', ($i+1))); // This one is important @@ -262,11 +268,11 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou // Add the clear event listener only for the thread $this->indexer->getEventDispatcher()->addSubscriber($this->clearEventSubscriber); - // Remove the limit so we'll keep on johnny walk'n - //$this->indexer->setOption('queue.size', -1); + echo $this->indexer->getQueue()->count(); // Seems to be a sub-process, ran it with a the number appended to the class - while ($this->indexer->getQueue()) { + while ($this->indexer->getQueue()->count()) { + echo $this->indexer->getQueue()->count(); $this->runInternal(sprintf('%s:%d', self::class, $argument->getProcessNumber()), $output); // Give them cores some relaxation From d6d5d4668aea56b667d821cd3cd4ea38b5056e8e Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Wed, 21 Dec 2016 10:20:43 +0100 Subject: [PATCH 35/48] Removed debug statements. --- Command/IndexerRunCommand.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index 8ae638e..8fbf22d 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -268,11 +268,8 @@ private function runProcess(ArgumentProcess $argument, InputInterface $input, Ou // Add the clear event listener only for the thread $this->indexer->getEventDispatcher()->addSubscriber($this->clearEventSubscriber); - echo $this->indexer->getQueue()->count(); - // Seems to be a sub-process, ran it with a the number appended to the class while ($this->indexer->getQueue()->count()) { - echo $this->indexer->getQueue()->count(); $this->runInternal(sprintf('%s:%d', self::class, $argument->getProcessNumber()), $output); // Give them cores some relaxation From 8a0801195357a7d37ffbafffeff76ef039f5bcab Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Wed, 21 Dec 2016 12:22:02 +0100 Subject: [PATCH 36/48] Clear should always be done. --- Command/IndexerQueueCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/IndexerQueueCommand.php b/Command/IndexerQueueCommand.php index 5627176..92952ec 100644 --- a/Command/IndexerQueueCommand.php +++ b/Command/IndexerQueueCommand.php @@ -186,7 +186,7 @@ protected function executeIndex(InputInterface $input, OutputInterface $output) // The entire site is going to be reindex so everything that is now in the queue // will be redone so just clear it so content is not double indexed. - //$this->getQueue()->clear(); + $this->getQueue()->clear(); } else { $builder->field('contentType')->in($input->getArgument('id')); From a1bcfaf26113bf6739bb7dc86694ca5c6c8058fc Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Wed, 21 Dec 2016 12:27:39 +0100 Subject: [PATCH 37/48] Added comment for -b mode. --- Command/IndexerRunCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index 8fbf22d..62f9b4a 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -114,7 +114,7 @@ protected function configure() 'blocking', 'b', InputOption::VALUE_NONE, - 0 + 'Block the current command until all sub-processes are done' ) ->setDescription('Execute a sol indexer run') ->setHelp(' From 2086cdae2b89e3e388cf5d2ea6165667b36bf950 Mon Sep 17 00:00:00 2001 From: Ger Jan Date: Tue, 10 Jan 2017 10:51:34 +0100 Subject: [PATCH 38/48] [INTEGRATED-946] Removed unused statements and fixed merge issue --- Command/IndexerRunCommand.php | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index 62f9b4a..fe3820e 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -19,18 +19,13 @@ use Integrated\Bundle\SolrBundle\Process\ProcessPoolGenerator; use Integrated\Common\Queue\Provider\DBAL\QueueProvider; -use Integrated\Common\Queue\Queue; use Integrated\Common\Solr\Indexer\Indexer; -use Integrated\Common\Solr\Indexer\IndexerInterface; - -use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -use Symfony\Component\HttpKernel\Kernel; use Symfony\Component\HttpKernel\KernelInterface; use Symfony\Component\Process\Process; use Symfony\Component\Filesystem\LockHandler; @@ -116,7 +111,7 @@ protected function configure() InputOption::VALUE_NONE, 'Block the current command until all sub-processes are done' ) - ->setDescription('Execute a sol indexer run') + ->setDescription('Execute a solr indexer run') ->setHelp(' The %command.name% command starts a indexer run. @@ -135,7 +130,7 @@ protected function execute(InputInterface $input, OutputInterface $output) if ($argument = $input->getArgument('processes')) { return $this->runProcess(new ArgumentProcess($argument), $input, $output); } else if ($input->getOption('full') || $input->getOption('daemon')) { - return $this->runExternal($input); + return $this->runExternal($input, $output); } return $this->runInternal(self::class, $output); @@ -176,9 +171,10 @@ private function runInternal($lock, OutputInterface $output) /** * @param InputInterface $input + * @param OutputInterface $output * @return int */ - private function runExternal(InputInterface $input) + private function runExternal(InputInterface $input, OutputInterface $output) { $wait = (int) $input->getOption('wait'); $wait = $wait * 1000; // convert from milli to micro @@ -191,16 +187,14 @@ private function runExternal(InputInterface $input) ); $process->setTimeout(0); - $process->run(function ($type, $buffer) use ($output) { - $output->write($buffer, false, $type); - }); + $process->run(); if (!$process->isSuccessful()) { break; // terminate when there is a error } if (!$input->getOption('daemon')) { - if ($this->indexer->getQueue()->count()) { + if (!$this->indexer->getQueue()->count()) { break; } } From 2b5ac2be93f3c2008759674bd68416bf9e336722 Mon Sep 17 00:00:00 2001 From: Ger Jan Date: Tue, 10 Jan 2017 10:55:42 +0100 Subject: [PATCH 39/48] [INTEGRATED-946] Restored output --- Command/IndexerRunCommand.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index fe3820e..95630d1 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -187,7 +187,9 @@ private function runExternal(InputInterface $input, OutputInterface $output) ); $process->setTimeout(0); - $process->run(); + $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer, false, $type); + }); if (!$process->isSuccessful()) { break; // terminate when there is a error From bb6c5e161314677839ebe2ddc7175fa77c21d6c8 Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Tue, 17 Jan 2017 17:32:15 +0100 Subject: [PATCH 40/48] Upgrade to symfony 2.8 --- Command/IndexerRunCommand.php | 4 +- DataCollector/SolariumDataCollector.php | 8 ++-- .../RegisterConfigFileProviderPass.php | 8 ++-- .../IntegratedSolrExtension.php | 11 +++--- .../DoctrineClearEventSubscriber.php | 2 +- IntegratedSolrBundle.php | 4 +- Resources/config/collector.xml | 8 +--- Resources/config/command.xml | 12 +----- Resources/config/converter.xml | 39 +++++-------------- .../{subscriber.xml => event_listeners.xml} | 9 +---- Resources/config/queue.xml | 29 +++++--------- Resources/config/solarium.xml | 7 ---- Resources/config/task.xml | 1 + Resources/config/types.xml | 26 ++++--------- .../TestObject.php} | 28 ++++++------- Tests/Solr/Type/FieldMapperTypeTest.php | 13 +------ composer.json | 2 +- 17 files changed, 67 insertions(+), 144 deletions(-) rename {EventSubscriber => EventListener}/DoctrineClearEventSubscriber.php (96%) rename Resources/config/{subscriber.xml => event_listeners.xml} (60%) rename Tests/{Solr/Type/FieldMapperTypeTestTestObject.php => Fixtures/TestObject.php} (78%) diff --git a/Command/IndexerRunCommand.php b/Command/IndexerRunCommand.php index 95630d1..bd70b90 100644 --- a/Command/IndexerRunCommand.php +++ b/Command/IndexerRunCommand.php @@ -13,8 +13,7 @@ use Exception; -use Integrated\Bundle\SolrBundle\EventSubscriber\DoctrineClearEventSubscriber; -use Integrated\Bundle\SolrBundle\EventSubscriber\MemoryEventSubscriber; +use Integrated\Bundle\SolrBundle\EventListener\DoctrineClearEventSubscriber; use Integrated\Bundle\SolrBundle\Process\ArgumentProcess; use Integrated\Bundle\SolrBundle\Process\ProcessPoolGenerator; @@ -159,7 +158,6 @@ private function runInternal($lock, OutputInterface $output) } $this->indexer->execute(); - } catch (Exception $e) { $output->writeln("Aborting: " . $e->getMessage()); diff --git a/DataCollector/SolariumDataCollector.php b/DataCollector/SolariumDataCollector.php index 342fc29..bc478a9 100644 --- a/DataCollector/SolariumDataCollector.php +++ b/DataCollector/SolariumDataCollector.php @@ -38,7 +38,7 @@ class SolariumDataCollector extends Plugin implements DataCollectorInterface, \S protected $startTime; /** - * @inheritdoc + * {@inheritdoc} */ protected function initPluginType() { @@ -48,7 +48,7 @@ protected function initPluginType() } /** - * @inheritdoc + * {@inheritdoc} */ public function collect(Request $request, Response $response, \Exception $exception = null) { @@ -115,7 +115,7 @@ public function getName() } /** - * @inheritdoc + * {@inheritdoc} */ public function serialize() { @@ -123,7 +123,7 @@ public function serialize() } /** - * @inheritdoc + * {@inheritdoc} */ public function unserialize($data) { diff --git a/DependencyInjection/CompilerPass/RegisterConfigFileProviderPass.php b/DependencyInjection/CompilerPass/RegisterConfigFileProviderPass.php index d87af92..363143c 100644 --- a/DependencyInjection/CompilerPass/RegisterConfigFileProviderPass.php +++ b/DependencyInjection/CompilerPass/RegisterConfigFileProviderPass.php @@ -11,14 +11,16 @@ namespace Integrated\Bundle\SolrBundle\DependencyInjection\CompilerPass; +use Integrated\Common\Converter\Config\Provider\XmlProvider; + use ReflectionClass; use Symfony\Component\Config\Resource\FileResource; - use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\Reference; +use Symfony\Component\Finder\Finder; /** * @author Jan Sanne Mulder @@ -72,7 +74,7 @@ protected function addProvider(ContainerBuilder $container, $dir, $bundle) return null; } - $definition = new Definition('%integrated_solr.converter.config.provider.file.class%'); + $definition = new Definition(XmlProvider::class); $definition->setPublic(false); $definition->setArguments([$this->addFinder($container, $dir . '/Resources/config/solr', $bundle)]); @@ -103,7 +105,7 @@ protected function addFinder(ContainerBuilder $container, $dir, $bundle) $container->addResource(new FileResource($dir)); // not really sure what this does - $definition = new Definition('%integrated_solr.converter.finder.class%'); + $definition = new Definition(Finder::class); $definition->setPublic(false); $definition->addMethodCall('in', [$dir]); diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 82d4833..8dbffbc 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -11,13 +11,14 @@ namespace Integrated\Bundle\SolrBundle\DependencyInjection; -use Symfony\Component\Config\FileLocator; +use Solarium\Client; +use Solarium\Core\Client\Endpoint; +use Symfony\Component\Config\FileLocator; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\Reference; use Symfony\Component\DependencyInjection\Loader\XmlFileLoader; - use Symfony\Component\HttpKernel\DependencyInjection\Extension; /** @@ -39,12 +40,12 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('converter.xml'); $loader->load('event.xml'); + $loader->load('event_listeners.xml'); $loader->load('command.xml'); $loader->load('indexer.xml'); $loader->load('queue.xml'); $loader->load('solarium.xml'); $loader->load('task.xml'); - $loader->load('subscriber.xml'); $loader->load('types.xml'); $loader->load('worker.xml'); @@ -61,7 +62,7 @@ public function load(array $configs, ContainerBuilder $container) $options['key'] = $name; $container->setDefinition( 'solarium.client.endpoint.' . $name, - new Definition('%integrated_solr.solarium.endpoint.class%', array($options)) + new Definition(Endpoint::class, array($options)) ); $endpoints[] = new Reference('solarium.client.endpoint.' . $name); @@ -70,7 +71,7 @@ public function load(array $configs, ContainerBuilder $container) $container->setDefinition( 'solarium.client', new Definition( - '%integrated_solr.solarium.client.class%', + Client::class, [ ['endpoint' => $endpoints], new Reference('integrated_solr.event.dispatcher') diff --git a/EventSubscriber/DoctrineClearEventSubscriber.php b/EventListener/DoctrineClearEventSubscriber.php similarity index 96% rename from EventSubscriber/DoctrineClearEventSubscriber.php rename to EventListener/DoctrineClearEventSubscriber.php index 59dece0..d7b035e 100644 --- a/EventSubscriber/DoctrineClearEventSubscriber.php +++ b/EventListener/DoctrineClearEventSubscriber.php @@ -9,7 +9,7 @@ * file that was distributed with this source code. */ -namespace Integrated\Bundle\SolrBundle\EventSubscriber; +namespace Integrated\Bundle\SolrBundle\EventListener; use Doctrine\ODM\MongoDB\DocumentManager; use Doctrine\ORM\EntityManager; diff --git a/IntegratedSolrBundle.php b/IntegratedSolrBundle.php index cb2ebb6..4b54987 100644 --- a/IntegratedSolrBundle.php +++ b/IntegratedSolrBundle.php @@ -26,7 +26,7 @@ class IntegratedSolrBundle extends Bundle { /** - * @inheritdoc + * {@inheritdoc} */ public function build(ContainerBuilder $container) { @@ -44,7 +44,7 @@ public function build(ContainerBuilder $container) } /** - * @inheritdoc + * {@inheritdoc} */ public function getContainerExtension() { diff --git a/Resources/config/collector.xml b/Resources/config/collector.xml index 831ce1c..fb7dd1f 100644 --- a/Resources/config/collector.xml +++ b/Resources/config/collector.xml @@ -4,15 +4,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Integrated\Bundle\SolrBundle\DataCollector\SolariumDataCollector - - - - + diff --git a/Resources/config/command.xml b/Resources/config/command.xml index bcfc84d..65e51fd 100644 --- a/Resources/config/command.xml +++ b/Resources/config/command.xml @@ -4,24 +4,16 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Integrated\Bundle\SolrBundle\Command\IndexerRunCommand - - - - - + - + %kernel.root_dir%/.. - diff --git a/Resources/config/converter.xml b/Resources/config/converter.xml index b713f6b..4f9e7ac 100644 --- a/Resources/config/converter.xml +++ b/Resources/config/converter.xml @@ -4,51 +4,30 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Integrated\Common\Converter\Type\ResolvedTypeFactory - - Integrated\Common\Converter\Type\RegistryInterface - Integrated\Common\Converter\Type\RegistryBuilder - - Symfony\Component\Finder\Finder - - Integrated\Common\Converter\Config\Provider\ChainProvider - Integrated\Common\Converter\Config\Provider\XmlProvider - - Integrated\Common\Converter\Config\ConfigResolver - - Integrated\Common\Converter\FilterContainerFactory - - Integrated\Common\Converter\Converter - - - - + - + - + + + - + - + - + - + diff --git a/Resources/config/subscriber.xml b/Resources/config/event_listeners.xml similarity index 60% rename from Resources/config/subscriber.xml rename to Resources/config/event_listeners.xml index 38e307f..f056c90 100644 --- a/Resources/config/subscriber.xml +++ b/Resources/config/event_listeners.xml @@ -4,18 +4,11 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Integrated\Bundle\SolrBundle\EventSubscriber\DoctrineClearEventSubscriber - - - - + - diff --git a/Resources/config/queue.xml b/Resources/config/queue.xml index 8cc5921..9cc6dd5 100644 --- a/Resources/config/queue.xml +++ b/Resources/config/queue.xml @@ -4,44 +4,31 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Integrated\Common\Queue\Provider\DBAL\Schema - Integrated\Common\Queue\Provider\DBAL\QueueProvider - Integrated\Common\Queue\QueueFactory - - Integrated\Common\Queue\Provider\Memory\QueueProvider - Integrated\Common\Queue\QueueFactory - - Integrated\Common\Queue\Queue - - - - + queue - + queue - + - + - + @@ -49,13 +36,15 @@ - + + solr-indexer - + + solr-worker diff --git a/Resources/config/solarium.xml b/Resources/config/solarium.xml index 7397047..0ab4d27 100644 --- a/Resources/config/solarium.xml +++ b/Resources/config/solarium.xml @@ -4,13 +4,6 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Solarium\Client - Solarium\Core\Client\Endpoint - - - diff --git a/Resources/config/task.xml b/Resources/config/task.xml index 831b9db..b34d55f 100644 --- a/Resources/config/task.xml +++ b/Resources/config/task.xml @@ -20,6 +20,7 @@ + Integrated\Bundle\ContentBundle\Document\Content\Content diff --git a/Resources/config/types.xml b/Resources/config/types.xml index 11fccc4..b953a6b 100644 --- a/Resources/config/types.xml +++ b/Resources/config/types.xml @@ -4,45 +4,33 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - - Integrated\Bundle\SolrBundle\Solr\Type\ClearType - Integrated\Bundle\SolrBundle\Solr\Type\CopyAppendType - Integrated\Bundle\SolrBundle\Solr\Type\CopyType - Integrated\Bundle\SolrBundle\Solr\Type\FieldAppendMapperType - Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType - Integrated\Bundle\SolrBundle\Solr\Type\RemoveType - Integrated\Bundle\SolrBundle\Solr\Type\JsonType - - - - + - + - + - + - + - + - + diff --git a/Tests/Solr/Type/FieldMapperTypeTestTestObject.php b/Tests/Fixtures/TestObject.php similarity index 78% rename from Tests/Solr/Type/FieldMapperTypeTestTestObject.php rename to Tests/Fixtures/TestObject.php index 62f8679..39cebbc 100644 --- a/Tests/Solr/Type/FieldMapperTypeTestTestObject.php +++ b/Tests/Fixtures/TestObject.php @@ -9,16 +9,18 @@ * file that was distributed with this source code. */ -namespace Integrated\Bundle\SolrBundle\Tests\Solr\Type; +namespace Integrated\Bundle\SolrBundle\Tests\Fixtures; + +use ArrayObject; +use DateTime; /** - * Class FieldMapperTypeTestTestObject - * @package Integrated\Bundle\SolrBundle\Tests\Solr\Type + * @author Jan Sanne Mulder */ -class FieldMapperTypeTestTestObject +class TestObject { /** - * @var \DateTime + * @var DateTime */ public $datetime; @@ -57,7 +59,7 @@ class FieldMapperTypeTestTestObject protected $field4 = 'field4'; /** - * @var \ArrayObject + * @var ArrayObject */ public $arrayObject; @@ -66,25 +68,25 @@ class FieldMapperTypeTestTestObject */ public function __construct() { - $this->datetime = new \DateTime('2014-01-01 00:30 CET'); + $this->datetime = new DateTime('2014-01-01 00:30 CET'); - $this->arrayObject = new \ArrayObject([ + $this->arrayObject = new ArrayObject([ 'field1' => 'field1', 'field2' => 'field2', 'field3' => 'field3', - 'array1' => new \ArrayObject([ + 'array1' => new ArrayObject([ 'field1' => 'array1.1', 'field2' => 'array1.2', 'field3' => 'array1.3' - ], \ArrayObject::ARRAY_AS_PROPS), + ], ArrayObject::ARRAY_AS_PROPS), - 'array2' => new \ArrayObject([ + 'array2' => new ArrayObject([ 'field1' => 'array2.1', 'field2' => 'array2.2', 'field3' => 'array2.3' - ], \ArrayObject::ARRAY_AS_PROPS) - ], \ArrayObject::ARRAY_AS_PROPS); + ], ArrayObject::ARRAY_AS_PROPS) + ], ArrayObject::ARRAY_AS_PROPS); } /** diff --git a/Tests/Solr/Type/FieldMapperTypeTest.php b/Tests/Solr/Type/FieldMapperTypeTest.php index e4f47fd..2828f53 100644 --- a/Tests/Solr/Type/FieldMapperTypeTest.php +++ b/Tests/Solr/Type/FieldMapperTypeTest.php @@ -11,25 +11,19 @@ namespace Integrated\Bundle\SolrBundle\Tests\Solr\Type; -use ArrayObject; -use DateTime; - use Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType; use Integrated\Common\Converter\Container; use Integrated\Common\Converter\ContainerInterface; -use Integrated\Bundle\SolrBundle\Tests\Solr\Type\FieldMapperTypeTestTestObject as TestObject; +use Integrated\Bundle\SolrBundle\Tests\Fixtures\TestObject; /** - * @covers Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType + * @covers \Integrated\Bundle\SolrBundle\Solr\Type\FieldMapperType * * @author Jan Sanne Mulder */ class FieldMapperTypeTest extends \PHPUnit_Framework_TestCase { - /** - * - */ public function testInterface() { self::assertInstanceOf('Integrated\\Common\\Converter\\Type\\TypeInterface', $this->getInstance()); @@ -242,9 +236,6 @@ public function buildStringConversionProvider() ]; } - /** - * - */ public function testGetName() { self::assertEquals('integrated.fields', $this->getInstance()->getName()); diff --git a/composer.json b/composer.json index 28ad986..88fa194 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ "prefer-stable": true, "require": { "php": ">=5.4", - "symfony/symfony": "~2.7", + "symfony/symfony": "~2.8", "integrated/library": "~0.6" }, "require-dev": { From 8e5db68de00b11a433857f789f854bfcd1428365 Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Fri, 20 Jan 2017 11:25:58 +0100 Subject: [PATCH 41/48] long to short array conversion --- DataCollector/SolariumDataCollector.php | 12 ++++++------ DependencyInjection/IntegratedSolrExtension.php | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/DataCollector/SolariumDataCollector.php b/DataCollector/SolariumDataCollector.php index bc478a9..ebb4fc1 100644 --- a/DataCollector/SolariumDataCollector.php +++ b/DataCollector/SolariumDataCollector.php @@ -30,7 +30,7 @@ class SolariumDataCollector extends Plugin implements DataCollectorInterface, \S /** * @var array */ - protected $data = array(); + protected $data = []; /** * @var float @@ -43,8 +43,8 @@ class SolariumDataCollector extends Plugin implements DataCollectorInterface, \S protected function initPluginType() { $dispatcher = $this->client->getEventDispatcher(); - $dispatcher->addListener(Events::PRE_EXECUTE_REQUEST, array($this, 'preExecuteRequest'), 1000); - $dispatcher->addListener(Events::POST_EXECUTE_REQUEST, array($this, 'postExecuteRequest'), -1000); + $dispatcher->addListener(Events::PRE_EXECUTE_REQUEST, [$this, 'preExecuteRequest'], 1000); + $dispatcher->addListener(Events::POST_EXECUTE_REQUEST, [$this, 'postExecuteRequest'], -1000); } /** @@ -74,12 +74,12 @@ public function preExecuteRequest(PreExecuteRequest $event) */ public function postExecuteRequest(PostExecuteRequest $event) { - $this->data['queries'][] = array( + $this->data['queries'][] = [ 'request' => $event->getRequest(), 'response' => $event->getResponse(), 'duration' => microtime(true) - $this->startTime, 'base_uri' => $event->getEndpoint()->getBaseUri(), - ); + ]; } /** @@ -87,7 +87,7 @@ public function postExecuteRequest(PostExecuteRequest $event) */ public function getQueries() { - return isset($this->data['queries']) ? $this->data['queries'] : array(); + return isset($this->data['queries']) ? $this->data['queries'] : []; } /** diff --git a/DependencyInjection/IntegratedSolrExtension.php b/DependencyInjection/IntegratedSolrExtension.php index 8dbffbc..122e2a5 100644 --- a/DependencyInjection/IntegratedSolrExtension.php +++ b/DependencyInjection/IntegratedSolrExtension.php @@ -56,13 +56,13 @@ public function load(array $configs, ContainerBuilder $container) $configuration = new Configuration(); $config = $this->processConfiguration($configuration, $configs); - $endpoints = array(); + $endpoints = []; foreach ($config['endpoints'] as $name => $options) { $options['key'] = $name; $container->setDefinition( 'solarium.client.endpoint.' . $name, - new Definition(Endpoint::class, array($options)) + new Definition(Endpoint::class, [$options]) ); $endpoints[] = new Reference('solarium.client.endpoint.' . $name); @@ -82,7 +82,7 @@ public function load(array $configs, ContainerBuilder $container) if ($container->getParameter('kernel.debug')) { $container->getDefinition('solarium.client')->addMethodCall( 'registerPlugin', - array('solarium.client.logger', new Reference('integrated_solr.solarium.data_collector')) + ['solarium.client.logger', new Reference('integrated_solr.solarium.data_collector')] ); } } From 8750dace63cc4801a45992c7c3c64c9d0cb023bd Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Thu, 26 Jan 2017 15:19:49 +0100 Subject: [PATCH 42/48] Added process locking. --- Command/WorkerCommand.php | 19 ++++++++++++++----- composer.json | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Command/WorkerCommand.php b/Command/WorkerCommand.php index d20f5df..3a6600c 100644 --- a/Command/WorkerCommand.php +++ b/Command/WorkerCommand.php @@ -16,7 +16,6 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -use Symfony\Component\Process\Process; use Symfony\Component\Filesystem\LockHandler; /** @@ -47,12 +46,22 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output) { - $worker = $this->getContainer()->get('integrated_solr.worker'); + $lock = new LockHandler(self::class); - if (null !== ($tasks = $input->getOption('tasks'))) { - $worker->setOption('tasks', intval($tasks)); + if (!$lock->lock()) { + return; } - $worker->execute(); + try { + $worker = $this->getContainer()->get('integrated_solr.worker'); + + if (null !== ($tasks = $input->getOption('tasks'))) { + $worker->setOption('tasks', intval($tasks)); + } + + $worker->execute(); + } finally { + $lock->release(); + } } } diff --git a/composer.json b/composer.json index 2061ddb..1829375 100644 --- a/composer.json +++ b/composer.json @@ -14,7 +14,7 @@ "minimum-stability": "dev", "prefer-stable": true, "require": { - "php": ">=5.4", + "php": ">=5.5", "symfony/symfony": "~2.4", "symfony/filesystem": "~2.6", "integrated/library": "~0.5" From da34125342048d91566dff7ea328f54a9aab2677 Mon Sep 17 00:00:00 2001 From: Marijn Otte Date: Wed, 8 Feb 2017 16:28:45 +0100 Subject: [PATCH 43/48] Allow username, password and scheme for solr --- DependencyInjection/Configuration.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 305099a..009b261 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -32,8 +32,11 @@ public function getConfigTreeBuilder() ->arrayNode('endpoints') ->prototype('array') ->children() + ->scalarNode('scheme')->defaultValue('http')->end() ->scalarNode('host')->defaultValue('localhost')->end() ->scalarNode('port')->defaultValue(8983)->end() + ->scalarNode('username')->defaultValue(null)->end() + ->scalarNode('password')->defaultValue(null)->end() ->scalarNode('path')->defaultValue('/solr')->end() ->scalarNode('core')->end() ->scalarNode('timeout')->defaultValue(5)->end() From 3aa1a395e731c59a33506ecf92913257c111dbd6 Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Thu, 2 Mar 2017 12:59:08 +0100 Subject: [PATCH 44/48] Upgrade the bundle from psr-0 to psr-4. --- composer.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index 88fa194..a20573a 100644 --- a/composer.json +++ b/composer.json @@ -23,9 +23,8 @@ "phpunit/phpunit": "4.2.*" }, "autoload":{ - "psr-0":{ - "Integrated\\Bundle\\SolrBundle": "" + "psr-4":{ + "Integrated\\Bundle\\SolrBundle\\": "" } - }, - "target-dir": "Integrated/Bundle/SolrBundle" + } } \ No newline at end of file From 97449eec23ec4642e2861e6bfb79e2c0fafbc5a9 Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Mon, 27 Mar 2017 11:33:47 +0200 Subject: [PATCH 45/48] added travis file --- .travis.yml | 7 +++++++ composer.json | 4 ++-- phpunit.xml.dist => phpunit.xml | 0 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .travis.yml rename phpunit.xml.dist => phpunit.xml (100%) diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9907926 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +install: +- curl -sS https://getcomposer.org/installer | php +before_script: +- php composer.phar update -n +script: +- vendor/bin/phpcs . --ignore=*/vendor/* --standard=PSR2 +- vendor/bin/phpunit -c phpunit.xml diff --git a/composer.json b/composer.json index 450b8b8..7f6c4a6 100644 --- a/composer.json +++ b/composer.json @@ -19,8 +19,8 @@ "integrated/library": "~0.6" }, "require-dev": { - "squizlabs/php_codesniffer": "2.*", - "phpunit/phpunit": "4.2.*" + "squizlabs/php_codesniffer": "^2.8", + "phpunit/phpunit": "^5.7" }, "autoload":{ "psr-4":{ diff --git a/phpunit.xml.dist b/phpunit.xml similarity index 100% rename from phpunit.xml.dist rename to phpunit.xml From 4c27562cbf01e650f90340c3cfca5dc18f2d10fa Mon Sep 17 00:00:00 2001 From: Johnny Borg Date: Thu, 30 Mar 2017 15:34:05 +0200 Subject: [PATCH 46/48] corrected directory --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9907926..1eb1ebe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,5 +3,5 @@ install: before_script: - php composer.phar update -n script: -- vendor/bin/phpcs . --ignore=*/vendor/* --standard=PSR2 +- vendor/bin/phpcs . --ignore=*/vendor/* --standard=PSR2 --extensions=php - vendor/bin/phpunit -c phpunit.xml From eae7f33ebe6dd3d3030c2bfc490445320ecca764 Mon Sep 17 00:00:00 2001 From: Jan Sanne Date: Tue, 11 Apr 2017 15:04:22 +0200 Subject: [PATCH 47/48] Updated symfony and integrated versions. --- composer.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 7f6c4a6..1116144 100644 --- a/composer.json +++ b/composer.json @@ -15,8 +15,9 @@ "prefer-stable": true, "require": { "php": ">=5.5", - "symfony/symfony": "~2.8", - "integrated/library": "~0.6" + "symfony/symfony": "~2.8 || ~3.0", + "integrated/library": "~0.7", + "integrated/content-bundle": "~0.7" }, "require-dev": { "squizlabs/php_codesniffer": "^2.8", From 8f6046273460a8032632ab0f0c14531e93f8bf57 Mon Sep 17 00:00:00 2001 From: Patrick Mestebeld Date: Tue, 25 Jul 2017 16:36:59 +0200 Subject: [PATCH 48/48] Fixed PHP error: Allowed memory size ... exhausted... --- Decorator/ContentProviderDetachDecorator.php | 50 +++++++++++ .../ContentTypeProviderDetachDecorator.php | 50 +++++++++++ Iterator/DetachIterator.php | 82 +++++++++++++++++++ Resources/config/task.xml | 16 +++- 4 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 Decorator/ContentProviderDetachDecorator.php create mode 100644 Decorator/ContentTypeProviderDetachDecorator.php create mode 100644 Iterator/DetachIterator.php diff --git a/Decorator/ContentProviderDetachDecorator.php b/Decorator/ContentProviderDetachDecorator.php new file mode 100644 index 0000000..6439024 --- /dev/null +++ b/Decorator/ContentProviderDetachDecorator.php @@ -0,0 +1,50 @@ + +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +namespace Integrated\Bundle\SolrBundle\Decorator; + +use Doctrine\Common\Persistence\ObjectManager; +use Integrated\Common\Solr\Task\Provider\ContentProviderInterface; +use Integrated\Bundle\SolrBundle\Iterator\DetachIterator; + +/** + * @author Patrick Mestebeld + */ +class ContentProviderDetachDecorator implements ContentProviderInterface +{ + /** + * @var ContentProviderInterface + */ + private $provider; + + /** + * @var ObjectManager + */ + private $manager; + + /** + * @param ContentProviderInterface $provider + * @param ObjectManager $manager + */ + public function __construct(ContentProviderInterface $provider, ObjectManager $manager) + { + $this->provider = $provider; + $this->manager = $manager; + } + + /** + * {@inheritdoc} + */ + public function getReferenced($id) + { + return new DetachIterator($this->provider->getReferenced($id), $this->manager); + } +} diff --git a/Decorator/ContentTypeProviderDetachDecorator.php b/Decorator/ContentTypeProviderDetachDecorator.php new file mode 100644 index 0000000..13ad9f9 --- /dev/null +++ b/Decorator/ContentTypeProviderDetachDecorator.php @@ -0,0 +1,50 @@ + +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +namespace Integrated\Bundle\SolrBundle\Decorator; + +use Doctrine\Common\Persistence\ObjectManager; +use Integrated\Common\Solr\Task\Provider\ContentTypeProviderInterface; +use Integrated\Bundle\SolrBundle\Iterator\DetachIterator; + +/** + * @author Patrick Mestebeld + */ +class ContentTypeProviderDetachDecorator implements ContentTypeProviderInterface +{ + /** + * @var ContentTypeProviderInterface + */ + private $provider; + + /** + * @var ObjectManager + */ + private $manager; + + /** + * @param ContentTypeProviderInterface $provider + * @param ObjectManager $manager + */ + public function __construct(ContentTypeProviderInterface $provider, ObjectManager $manager) + { + $this->provider = $provider; + $this->manager = $manager; + } + + /** + * {@inheritdoc} + */ + public function getContent($id) + { + return new DetachIterator($this->provider->getContent($id), $this->manager); + } +} diff --git a/Iterator/DetachIterator.php b/Iterator/DetachIterator.php new file mode 100644 index 0000000..5fff24a --- /dev/null +++ b/Iterator/DetachIterator.php @@ -0,0 +1,82 @@ + +* +* For the full copyright and license information, please view the LICENSE +* file that was distributed with this source code. +*/ + +namespace Integrated\Bundle\SolrBundle\Iterator; + +use Doctrine\Common\Persistence\ObjectManager; +use Iterator; + +/** + * @author Patrick Mestebeld + */ +class DetachIterator implements Iterator +{ + /** + * @var ObjectManager + */ + private $manager; + + /** + * @var Iterator + */ + private $iterator; + + /** + * @param Iterator $iterator + * @param ObjectManager $manager + */ + public function __construct(Iterator $iterator, ObjectManager $manager) + { + $this->iterator = $iterator; + $this->manager = $manager; + } + + /** + * {@inheritdoc} + */ + public function current() + { + $this->manager->detach($current = $this->iterator->current()); + return $current; + } + + /** + * {@inheritdoc} + */ + public function next() + { + $this->iterator->next(); + } + + /** + * {@inheritdoc} + */ + public function key() + { + return $this->iterator->key(); + } + + /** + * {@inheritdoc} + */ + public function valid() + { + return $this->iterator->valid(); + } + + /** + * {@inheritdoc} + */ + public function rewind() + { + $this->iterator->rewind(); + } +} diff --git a/Resources/config/task.xml b/Resources/config/task.xml index b34d55f..ed500c3 100644 --- a/Resources/config/task.xml +++ b/Resources/config/task.xml @@ -28,8 +28,18 @@ - + + + + + + + + + + + @@ -37,7 +47,7 @@ - + @@ -46,4 +56,4 @@ - \ No newline at end of file +