Merge pull request #55 from valantic/feature/es8

v4 / Elasticsearch 8

.github/workflows/php.yml

@@ -17,7 +17,6 @@ jobs:
         php: [ '8.1', '8.2' ]
         pimcore: [ '^11.0' ]
         stability: [ prefer-lowest, prefer-stable ]
-        elastica: [ '^7.1', '8.x-dev#0f6b04b' ]
 
     steps:
       - name: Checkout code
@@ -34,7 +33,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          composer require "pimcore/pimcore:${{ matrix.pimcore }}" "ruflin/elastica:${{ matrix.elastica }}" --no-interaction --no-update
+          composer require "pimcore/pimcore:${{ matrix.pimcore }}" --no-interaction --no-update
           composer update --${{ matrix.stability }} --prefer-dist --no-interaction
 
       - name: List installed dependencies

README.md

@@ -12,8 +12,8 @@ The only job of the bundle is to store Pimcore elements (assets, documents, data
 
 1. `composer require valantic/pimcore-elastica-bridge`
 1. Edit `config/bundles.php` and add `\Valantic\ElasticaBridgeBundle\ValanticElasticaBridgeBundle::class => ['all' => true],`
-1. Configure the connection to your Elasticsearch cluster as seen in [`example/app/config/config.yml`](example/app/config/config.yml)
-1. Don't forget to register your newly created services (implementing `IndexInterface` etc.) in your `services.yml`
+1. Configure the connection to your Elasticsearch cluster as seen in [`example/app/config/config.yaml`](example/app/config/config.yaml)
+1. Don't forget to register your newly created services (implementing `IndexInterface` etc.) in your `services.yaml`
    ```yml
    App\Elasticsearch\:
    resource: '../../Elasticsearch'
@@ -53,11 +53,25 @@ See the [`ProductIndexDocument` provided in the example](docs/example/src/Elasti
 ```yaml
 valantic_elastica_bridge:
     client:
-        host:                 localhost
-        port:                 9200
-        addSentryBreadcrumbs: false
+
+        # The DSN to connect to the Elasticsearch cluster.
+        dsn:                  'http://localhost:9200'
+
+        # If true, breadcrumbs are added to Sentry for every request made to Elasticsearch via Elastica.
+        should_add_sentry_breadcrumbs: false
+    indexing:
+
+        # To prevent overlapping indexing jobs. Set to a value higher than the slowest index. Value is specified in seconds.
+        lock_timeout:         300
+
+        # If true, when a document fails to be indexed, it will be skipped and indexing continue with the next document. If false, indexing that index will be aborted.
+        should_skip_failing_documents: false
 ```
 
+## Queue
+
+[Set up a worker](https://symfony.com/doc/current/messenger.html#consuming-messages-running-the-worker) to process `elastica_bridge_index`. Alternatively you can route the transport to use the `sync` handler: `framework.messenger.transports.elastica_bridge_index: 'sync'`.
+
 ## Indexing
 
 ### Bulk
@@ -86,6 +100,8 @@ The bridge automatically listens to Pimcore events and updates documents as need
 
 This can be globally disabled by calling `\Valantic\ElasticaBridgeBundle\EventListener\Pimcore\ChangeListener::disableListener();`.
 
+You can also dispatch a `Valantic\ElasticaBridgeBundle\Messenger\Message\RefreshElement` message to handle updates to related objects which are not triggered by the `ChangeListener`.
+
 ## Status
 
 ```

UPGRADE.md

@@ -1,14 +1,26 @@
-# Upgrade from v1 to v2
+# UPGRADE
 
+## Upgrade from v3 to v4
 
-## Migration
+- Remove deprecated options `valantic_elastica_bridge.client.host` and `valantic_elastica_bridge.client.port`. Use `valantic_elastica_bridge.client.dsn` instead, e.g. `http://localhost:9200`
+- Renamed `valantic_elastica_bridge.client.addSentryBreadcrumbs` to `valantic_elastica_bridge.client.should_add_sentry_breadcrumbs`
+- See [README#Queue](./README#queue) to set up the **required** Symfony Messenger workers.
+
+## Upgrade from v2 to v3
+
+- no code changes necessary
+
+## Upgrade from v1 to v2
+
+
+### Migration
 
 - `IndexDocumentInterface` implementations should now extend `\Valantic\ElasticaBridgeBundle\Document\AbstractDocument`. `getType()` should now return one of `\Valantic\ElasticaBridgeBundle\Enum\DocumentType`
 - `Valantic\ElasticaBridgeBundle\DocumentType\Index\ListingTrait` was removed, remove any references to it [#30](https://github.com/valantic/pimcore-elastica-bridge/issues/30)
 - Update references to renamed classes and interfaces (see next section)
 - see also the example in [`docs/example/`](./docs/example/)
 
-## Breaking Changes
+### Breaking Changes
 
 - PHP 8.1+ [#26](https://github.com/valantic/pimcore-elastica-bridge/issues/26)
 - `\Valantic\ElasticaBridgeBundle\EventListener\Pimcore\AbstractListener` was renamed to `\Valantic\ElasticaBridgeBundle\EventListener\Pimcore\ChangeListener`
@@ -23,14 +35,14 @@
 - `Valantic\ElasticaBridgeBundle\Document\TenantAwareTrait` has been replaced by `Valantic\ElasticaBridgeBundle\Document\AbstractTenantAwareDocument`
 - `Valantic\ElasticaBridgeBundle\DocumentType\Index\DataObjectNormalizerTrait` was moved to `Valantic\ElasticaBridgeBundle\Document\DataObjectNormalizerTrait`
 
-## New Features
+### New Features
 
 - PHPStan generics annotations for `\Valantic\ElasticaBridgeBundle\Document\DocumentInterface` and related helper traits [#32](https://github.com/valantic/pimcore-elastica-bridge/issues/32)
 - Added `\Valantic\ElasticaBridgeBundle\Service\PropagateChanges::handle` to programmatically update an element in all indices [#33](https://github.com/valantic/pimcore-elastica-bridge/issues/33)
 - Added support for assets [#34](https://github.com/valantic/pimcore-elastica-bridge/issues/34)
 - Allow `\Valantic\ElasticaBridgeBundle\Document\DocumentInterface::getSubType` to return `null` for generic, element-level indices [#42](https://github.com/valantic/pimcore-elastica-bridge/issues/42)
 
-## Other changes
+### Other changes
 
 - `:cleanup` now defaults to only cleaning up bundle indices [#27](https://github.com/valantic/pimcore-elastica-bridge/issues/27)
 - Removed `--check` from `:index` [#41](https://github.com/valantic/pimcore-elastica-bridge/issues/41)

composer.json

@@ -8,7 +8,7 @@
     "ext-json": "*",
     "pimcore/pimcore": "^11.0",
     "psr/log": "^3.0",
-    "ruflin/elastica": "^7.1 || 8.x-dev#0f6b04b",
+    "ruflin/elastica": "8.x-dev",
     "symfony/console": "^6.2",
     "symfony/lock": "^6.2"
   },
@@ -18,7 +18,7 @@
     "phpstan/phpstan": "^1.10.58",
     "phpstan/phpstan-deprecation-rules": "^1.1.4",
     "phpstan/phpstan-strict-rules": "^1.5.2",
-    "rector/rector": "^0.18.13",
+    "rector/rector": "^1.0.1",
     "roave/security-advisories": "dev-latest",
     "sentry/sentry": "^3.22.1",
     "symfony/http-client": "^6.4.3"

docs/example/config/config.yaml

@@ -0,0 +1,3 @@
+valantic_elastica_bridge:
+  client:
+    dsn: 'http://localhost:9200'

rector.php

@@ -2,30 +2,39 @@
 
 declare(strict_types=1);
 
+use Rector\CodeQuality\Rector\Foreach_\SimplifyForeachToCoalescingRector;
+use Rector\CodeQuality\Rector\FuncCall\SingleInArrayToCompareRector;
 use Rector\CodingStyle\Rector\FuncCall\CountArrayToEmptyArrayComparisonRector;
 use Rector\Config\RectorConfig;
 use Rector\Php55\Rector\String_\StringClassNameToClassConstantRector;
 use Rector\Php80\Rector\FunctionLike\MixedTypeRector;
-use Rector\Set\ValueObject\LevelSetList;
-use Rector\Set\ValueObject\SetList;
 
-return static function (RectorConfig $rectorConfig): void {
-    $rectorConfig->paths([
+return RectorConfig::configure()
+    ->withPhpSets()
+    ->withPreparedSets(
+        codeQuality: true,
+    )
+    ->withAttributesSets(
+        symfony: true,
+        doctrine: true,
+    )
+    ->withPaths([
         __DIR__ . '/src',
-    ]);
-
-    $rectorConfig->skip([
+    ])
+    ->withSkip([
         CountArrayToEmptyArrayComparisonRector::class,
         StringClassNameToClassConstantRector::class => [
             'src/Elastica/Client/ElasticsearchClientFactory.php',
         ],
         MixedTypeRector::class => [
             'src/Document/DataObjectNormalizerTrait.php',
+            'src/Index/IndexInterface.php',
         ],
-    ]);
-
-    $rectorConfig->sets([
-        LevelSetList::UP_TO_PHP_81,
-        SetList::CODE_QUALITY,
-    ]);
-};
+        SingleInArrayToCompareRector::class => [
+            'src/Command/NonBundleIndexTrait.php',
+        ],
+        SimplifyForeachToCoalescingRector::class => [
+            'src/Repository/IndexRepository.php',
+        ],
+    ])
+    ->withRootFiles();

src/Command/Cleanup.php

@@ -4,7 +4,7 @@
 
 namespace Valantic\ElasticaBridgeBundle\Command;
 
-use Elastica\Exception\ResponseException;
+use Elastic\Elasticsearch\Exception\ElasticsearchException;
 use Symfony\Component\Console\Helper\QuestionHelper;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -15,6 +15,7 @@
 
 class Cleanup extends BaseCommand
 {
+    use NonBundleIndexTrait;
     private const OPTION_ALL_IN_CLUSTER = 'all';
 
     public function __construct(
@@ -54,15 +55,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $indices = $this->getIndices();
 
         foreach ($indices as $index) {
+            if (!$this->shouldProcessNonBundleIndex($index)) {
+                continue;
+            }
+
             $client = $this->esClient->getIndex($index);
 
+            if ($client->getSettings()->getBool('hidden')) {
+                continue;
+            }
+
             foreach ($client->getAliases() as $alias) {
                 $client->removeAlias($alias);
             }
 
             try {
                 $client->delete();
-            } catch (ResponseException $e) {
+            } catch (ElasticsearchException $e) {
                 $this->output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
             }
         }
@@ -81,7 +90,7 @@ private function getIndices(): array
 
         $indices = [];
 
-        foreach ($this->indexRepository->flattened() as $indexConfig) {
+        foreach ($this->indexRepository->flattenedAll() as $indexConfig) {
             if ($indexConfig->usesBlueGreenIndices()) {
                 $indices[] = $indexConfig->getBlueGreenActiveElasticaIndex()->getName();
                 $indices[] = $indexConfig->getBlueGreenInactiveElasticaIndex()->getName();

src/Command/Index.php

@@ -5,22 +5,20 @@
 namespace Valantic\ElasticaBridgeBundle\Command;
 
 use Elastica\Index as ElasticaIndex;
-use Elastica\Request;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\ConsoleOutput;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\HttpKernel\KernelInterface;
-use Symfony\Component\Lock\LockFactory;
-use Symfony\Component\Lock\LockInterface;
 use Symfony\Component\Process\Process;
 use Valantic\ElasticaBridgeBundle\Elastica\Client\ElasticsearchClient;
 use Valantic\ElasticaBridgeBundle\Enum\IndexBlueGreenSuffix;
 use Valantic\ElasticaBridgeBundle\Exception\Index\BlueGreenIndicesIncorrectlySetupException;
 use Valantic\ElasticaBridgeBundle\Index\IndexInterface;
-use Valantic\ElasticaBridgeBundle\Repository\ConfigurationRepository;
 use Valantic\ElasticaBridgeBundle\Repository\IndexRepository;
+use Valantic\ElasticaBridgeBundle\Service\LockService;
+use Valantic\ElasticaBridgeBundle\Util\ElasticsearchResponse;
 
 class Index extends BaseCommand
 {
@@ -34,8 +32,7 @@ public function __construct(
         private readonly IndexRepository $indexRepository,
         private readonly ElasticsearchClient $esClient,
         private readonly KernelInterface $kernel,
-        private readonly LockFactory $lockFactory,
-        private readonly ConfigurationRepository $configurationRepository,
+        private readonly LockService $lockService,
     ) {
         parent::__construct();
     }
@@ -73,7 +70,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $skippedIndices = [];
 
-        foreach ($this->indexRepository->flattened() as $indexConfig) {
+        foreach ($this->indexRepository->flattenedAll() as $indexConfig) {
             if (
                 is_array($this->input->getArgument(self::ARGUMENT_INDEX))
                 && count($this->input->getArgument(self::ARGUMENT_INDEX)) > 0
@@ -84,7 +81,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 continue;
             }
 
-            $lock = $this->getLock($indexConfig);
+            $lock = $this->lockService->getIndexingLock($indexConfig);
 
             if (!$lock->acquire()) {
                 if ($this->input->getOption(self::OPTION_LOCK_RELEASE) === true) {
@@ -229,7 +226,7 @@ private function ensureCorrectBlueGreenIndexSetup(IndexInterface $indexConfig):
         // In case an index with the same name as the blue/green alias exists, delete it
         if (
             $nonAliasIndex->exists()
-            && !$this->esClient->request('_alias/' . $indexConfig->getName(), Request::HEAD)->isOk()
+            && !ElasticsearchResponse::getResponse($this->esClient->indices()->existsAlias(['name' => $indexConfig->getName()]))->asBool()
         ) {
             $nonAliasIndex->delete();
             $this->output->writeln('<comment>-> Deleted non-blue/green index to prepare for blue/green usage</comment>');
@@ -259,13 +256,4 @@ private function ensureCorrectBlueGreenIndexSetup(IndexInterface $indexConfig):
 
         $this->output->writeln('<comment>-> Ensured indices are correctly set up with alias</comment>');
     }
-
-    private function getLock(mixed $indexConfig): LockInterface
-    {
-        return $this->lockFactory
-            ->createLock(
-                __METHOD__ . '->' . $indexConfig->getName(),
-                ttl: $this->configurationRepository->getIndexingLockTimeout()
-            );
-    }
 }

src/Command/NonBundleIndexTrait.php

@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Valantic\ElasticaBridgeBundle\Command;
+
+trait NonBundleIndexTrait
+{
+    protected function shouldProcessNonBundleIndex(string $name): bool
+    {
+        return !in_array($name, ['.geoip_databases'], true);
+    }
+}