diff --git a/composer.json b/composer.json index b0e7e782..f1ef5087 100644 --- a/composer.json +++ b/composer.json @@ -43,6 +43,10 @@ "ezsystems/ezplatform-code-style": "^0.1.0", "mikey179/vfsstream": "^1.6" }, + "suggest": { + "aws/aws-sdk-php": "For compiled schema synchronization over AWS-S3", + "ext-redis": "*" + }, "autoload": { "psr-4": { "EzSystems\\EzPlatformGraphQL\\": "src", diff --git a/src/Command/PublishSchemaCommand.php b/src/Command/PublishSchemaCommand.php new file mode 100644 index 00000000..60f197c4 --- /dev/null +++ b/src/Command/PublishSchemaCommand.php @@ -0,0 +1,60 @@ +definitionsDirectory = $definitionsDirectory; + $this->sharedSchema = $sharedSchema; + } + + public function configure() + { + $this->setName('ibexa:graphql:publish-schema'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $io = new SymfonyStyle($input, $output); + + foreach ((new Finder())->files()->in($this->definitionsDirectory) as $file) { + $this->sharedSchema->addFile($file->getBasename(), $file->getContents()); + } + + try { + $this->sharedSchema->publish( + filemtime("$this->definitionsDirectory/__classes.map") + ); + } catch (\Exception $e) { + $io->error($e->getMessage()); + return self::FAILURE; + } + + return self::SUCCESS; + } +} diff --git a/src/DependencyInjection/EzSystemsEzPlatformGraphQLExtension.php b/src/DependencyInjection/EzSystemsEzPlatformGraphQLExtension.php index 6076f2c9..18282b39 100644 --- a/src/DependencyInjection/EzSystemsEzPlatformGraphQLExtension.php +++ b/src/DependencyInjection/EzSystemsEzPlatformGraphQLExtension.php @@ -6,6 +6,7 @@ */ namespace EzSystems\EzPlatformGraphQL\DependencyInjection; +use Aws\S3\S3Client; use EzSystems\EzPlatformGraphQL\DependencyInjection\GraphQL\YamlSchemaProvider; use Symfony\Component\Config\FileLocator; use Symfony\Component\DependencyInjection\ContainerBuilder; @@ -39,8 +40,16 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('services/mutations.yaml'); $loader->load('services/resolvers.yaml'); $loader->load('services/schema.yaml'); + $loader->load('services/schema_sync.yaml'); $loader->load('services/services.yaml'); $loader->load('default_settings.yaml'); + + if (extension_loaded('redis')) { + $loader->load('services/schema_sync.yaml'); + if (class_exists(S3Client::class)) { + $loader->load('services/schema_sync_s3.yaml'); + } + } } /** diff --git a/src/Resources/config/services/schema_sync.yaml b/src/Resources/config/services/schema_sync.yaml new file mode 100644 index 00000000..d712092c --- /dev/null +++ b/src/Resources/config/services/schema_sync.yaml @@ -0,0 +1,39 @@ +parameters: + ibexa_graphql.definitions_directory: '%kernel.cache_dir%/overblog/graphql-bundle/__definitions__' + ibexa_graphql.sync.shared_directory: '/tmp/graphql-schema' + +services: + _defaults: + autoconfigure: true + autowire: true + public: false + bind: + EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler: '@EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler' + Redis $graphQLSyncRedis: '@ibexa_graphql.sync.redis_client' + + EzSystems\EzPlatformGraphQL\Command\PublishSchemaCommand: + arguments: + $definitionsDirectory: '%ibexa_graphql.definitions_directory%' + + EzSystems\EzPlatformGraphQL\Schema\Sync\LocalFolderSharedSchema: + arguments: + $sharedDirectory: '%ibexa_graphql.sync.shared_directory%' + + EzSystems\EzPlatformGraphQL\Schema\Sync\SharedSchema: '@EzSystems\EzPlatformGraphQL\Schema\Sync\LocalFolderSharedSchema' + + EzSystems\EzPlatformGraphQL\Schema\Sync\AddTypesSolutions: + arguments: + $definitionsDirectory: '%ibexa_graphql.definitions_directory%' + + EzSystems\EzPlatformGraphQL\Schema\Sync\UpdateSchemaIfNeeded: + arguments: + $definitionsDirectory: '%ibexa_graphql.definitions_directory%' + + ibexa_graphql.sync.redis_client: + class: Redis + calls: + - connect: ['%env(REDIS_GRAPHQL_HOST)%', '%env(REDIS_GRAPHQL_PORT)%'] + - select: ['%env(REDIS_GRAPHQL_DBINDEX)%'] + + EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler: ~ + diff --git a/src/Resources/config/services/schema_sync_s3.yaml b/src/Resources/config/services/schema_sync_s3.yaml new file mode 100644 index 00000000..e241d209 --- /dev/null +++ b/src/Resources/config/services/schema_sync_s3.yaml @@ -0,0 +1,17 @@ +services: + _defaults: + bind: + Aws\S3\S3Client $graphQLSyncS3Client: '@ibexa_graphql.sync.s3_client' + EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler: '@EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler' + + ibexa_graphql.sync.s3_client: + class: Aws\S3\S3Client + arguments: + - version: '2006-03-01' + region: '%env(GRAPHQL_SYNC_S3_REGION)%' + + EzSystems\EzPlatformGraphQL\Schema\Sync\S3SharedSchema: + arguments: + $bucket: '%env(GRAPHQL_SYNC_S3_BUCKET)%' + + EzSystems\EzPlatformGraphQL\Schema\Sync\SharedSchema: '@EzSystems\EzPlatformGraphQL\Schema\Sync\S3SharedSchema' diff --git a/src/Schema/Sync/AddTypesSolutions.php b/src/Schema/Sync/AddTypesSolutions.php new file mode 100644 index 00000000..26a4b7b6 --- /dev/null +++ b/src/Schema/Sync/AddTypesSolutions.php @@ -0,0 +1,88 @@ +typeResolver = $typeResolver; + $this->configProcessor = $configProcessor; + $this->globalVariables = $globalVariables; + $this->definitionsDirectory = $definitionsDirectory; + $this->logger = $logger; + } + + /** + * @inheritDoc + */ + public static function getSubscribedEvents() + { + return [Events::PRE_EXECUTOR => 'registerTypes']; + } + + public function registerTypes(ExecutorArgumentsEvent $event) + { + $classMapFile = "$this->definitionsDirectory/__classes.map"; + $map = include($classMapFile); + ksort($map); + foreach ($map as $class => $file) { + $typeName = str_replace('Overblog\\GraphQLBundle\\__DEFINITIONS__\\', '', $class); + $typeName = substr($typeName, 0, -4); + if ($this->typeResolver->hasSolution($class)) { + continue; + } + $this->typeResolver->addSolution( + $class, + [ + [$this, 'build'], + [$class] + ], + [$typeName], + [ + 'id' => $class, + 'aliases' => [$typeName], + 'alias' => $typeName, + 'generated' => true + ] + ); + } + } + + public function build($id) + { + return new $id($this->configProcessor, $this->globalVariables); + } +} diff --git a/src/Schema/Sync/LocalFolderSharedSchema.php b/src/Schema/Sync/LocalFolderSharedSchema.php new file mode 100644 index 00000000..0a3e8299 --- /dev/null +++ b/src/Schema/Sync/LocalFolderSharedSchema.php @@ -0,0 +1,68 @@ + file contents + */ + private $files = []; + + /** + * @var \EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler + */ + private $timestampHandler; + + public function __construct(TimestampHandler $timestampHandler, string $sharedDirectory) + { + $this->sharedDirectory = rtrim($sharedDirectory, '/'); + $this->timestampHandler = $timestampHandler; + } + + public function addFile(string $name, string $contents) + { + $this->files[$name] = $contents; + } + + public function publish(int $timestamp) + { + $fs = new Filesystem(); + + $targetDirectory = "$this->sharedDirectory/$timestamp"; + $fs->mkdir($targetDirectory); + + foreach ($this->files as $name => $contents) { + $fs->dumpFile("$targetDirectory/$name", $contents); + } + + $this->timestampHandler->set($timestamp); + } + + public function getFiles(int $timestamp): array + { + $directory = "$this->sharedDirectory/$timestamp"; + if (!file_exists($directory) || !is_dir($directory)) { + throw new \Exception("Directory not found"); + } + + $files = []; + foreach ((new Finder())->files()->in($directory) as $file) { + $files[$file->getBasename()] = $file->getContents(); + } + + return $files; + } +} diff --git a/src/Schema/Sync/README.md b/src/Schema/Sync/README.md new file mode 100644 index 00000000..33d84c7c --- /dev/null +++ b/src/Schema/Sync/README.md @@ -0,0 +1,68 @@ +# GraphQL schema sync + +An experimental mechanism for publishing a compiled schema so that secondary servers can pull it and install it without recompiling their own containe. + +## Configuration + +### Redis +The feature requires Redis for publishing the latest schema timestamp (it is suggested in `composer.json`). + +The feature comes with a default redis client service, `ibexa_graphql.sync.redis_client`. +It is configured using the following environment variables: +```.dotenv +REDIS_GRAPHQL_HOST=1.2.3.4 +REDIS_GRAPHQL_PORT=6379 +REDIS_GRAPHQL_DBINDEX=0 +``` + +If you want to use your own client, you can redefined the service with the same name in your +project's services definitions. + +If you already have your own and want to re-use it, create an alias with that name: +```yaml +# config/services.yaml +services: + ibexa_graphql.sync.redis_client: '@app.redis_client' +``` + +### AWS S3 +Amazon S3 can be used to publish the schema files. To enable it, make sure that `aws/aws-sdk-php` +is installed on your project. + +It uses a default client service named `ibexa_graphql.sync.s3_client`, based on the default +environment variables expected by the SDK: + +```dotenv +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +``` + +If you already have an s3 client service, alias it to `ibexa_graphql.sync.s3_client`: +```yaml +# config/services.yaml +services: + ibexa_graphql.sync.s3_client: '@app.s3_client' +``` + +The feature also requires two extra settings for the bucket and the region: +```dotenv +GRAPHQL_SYNC_S3_BUCKET=ibexa-graphql +GRAPHQL_SYNC_S3_REGION=eu-west-1 +``` + +## Usage +One server will do the schema generation + compiling, and run a command +(`ibexa:graphql:publish-schema`) to publish the schema. The command can be executed during a deployment process, +or manually. + +Publishing will: + +1. push the compiled schema (`%kernel.cache_dir%/overblog/graphql-bundle/__definitions__/*`) to a SharedSchema +2. set the published schema timestamp using a TimestampHandler. + +Secondary servers, when a GraphQL query is executed (`UpdateSchemaIfNeeded` subscriber), will compare the timestamp to +theirs (modification time of `__classes.map`). If the remote schema is newer, it will be pulled and installed on shutdown. + +Since the graphql schema types are compiled into the container as services, types that were added to the published schema +(new content types, etc) need to be registered on runtime. This is done by the `Schema\Sync\AddTypesSolutions` subscriber. +It checks which of the type classes do not have a solution in the current schema, and adds them to it. diff --git a/src/Schema/Sync/RedisTimestampHandler.php b/src/Schema/Sync/RedisTimestampHandler.php new file mode 100644 index 00000000..271a56b2 --- /dev/null +++ b/src/Schema/Sync/RedisTimestampHandler.php @@ -0,0 +1,38 @@ +redis = $graphQLSyncRedis; + $this->key = $key; + } + + public function set($timestamp) + { + $this->redis->set($this->key, $timestamp); + } + + public function get(): int + { + return $this->redis->get($this->key); + } +} diff --git a/src/Schema/Sync/S3SharedSchema.php b/src/Schema/Sync/S3SharedSchema.php new file mode 100644 index 00000000..97c20a89 --- /dev/null +++ b/src/Schema/Sync/S3SharedSchema.php @@ -0,0 +1,97 @@ + file contents + */ + private $files = []; + + /** + * @var \EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler + */ + private $timestampHandler; + /** + * @var \Aws\S3\S3Client + */ + private $s3; + + private $bucket; + + public function __construct(TimestampHandler $timestampHandler, S3Client $graphQLSyncS3Client, string $bucket) + { + $this->timestampHandler = $timestampHandler; + $this->s3 = $graphQLSyncS3Client; + $this->bucket = $bucket; + } + + public function addFile(string $name, string $contents) + { + $this->files[$name] = $contents; + } + + public function publish(int $timestamp) + { + foreach ($this->files as $name => $contents) { + $this->putFileToS3("$timestamp/$name", $contents); + } + + $this->timestampHandler->set($timestamp); + } + + public function getFiles(int $timestamp): array + { + if (!$this->hasFileOnS3("$timestamp/__classes.map")) { + throw new \Exception("Shared schema not found"); + } + + $files = []; + + $prefix = "$timestamp/"; + $listResult = $this->s3->listObjectsV2(['Bucket' => $this->bucket, 'Prefix' => $prefix]); + foreach ($listResult->get('Contents') as $listItem) { + $fileResult = $this->s3->getObject(['Bucket' => $this->bucket, 'Key' => $listItem['Key']]); + $fileName = str_replace($prefix, '', $listItem['Key']); + $files[$fileName] = $fileResult->get('Body')->getContents(); + } + + return $files; + } + + private function putFileToS3(string $name, string $contents): void + { + try { + $this->s3->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $name, + 'Body' => $contents, + ]); + } catch (S3Exception $e) { + throw new \Exception("Error creating file", 0, $e); + } + } + + private function hasFileOnS3(string $path): bool + { + try { + $this->s3->getObject(['Bucket' => $this->bucket, 'Key' => $path]); + } catch (S3Exception $e) { + return false; + } + + return true; + } +} diff --git a/src/Schema/Sync/SharedSchema.php b/src/Schema/Sync/SharedSchema.php new file mode 100644 index 00000000..74dffabb --- /dev/null +++ b/src/Schema/Sync/SharedSchema.php @@ -0,0 +1,27 @@ + file contents + * @throws \Exception if the directory is not found + */ + public function getFiles(int $timestamp); +} diff --git a/src/Schema/Sync/TimestampHandler.php b/src/Schema/Sync/TimestampHandler.php new file mode 100644 index 00000000..1a38009e --- /dev/null +++ b/src/Schema/Sync/TimestampHandler.php @@ -0,0 +1,14 @@ +lockFactory = $lockFactory; + $this->logger = $graphqlLogger; + $this->definitionsDirectory = $definitionsDirectory; + $this->timestampHandler = $timestampHandler; + $this->sharedSchema = $sharedSchema; + } + + /** + * @inheritDoc + */ + public static function getSubscribedEvents() + { + return [Events::PRE_EXECUTOR => ['updateSchema', 10]]; + } + + public function updateSchema(ExecutorArgumentsEvent $event) + { + $lock = $this->lockFactory->createLock('graphql_schema_sync'); + if (!$lock->acquire()) { + return; + } + + $localSchemaTimestamp = filemtime("$this->definitionsDirectory/__classes.map"); + $remoteSchemaTimestamp = $this->timestampHandler->get(); + if ($remoteSchemaTimestamp === false || $remoteSchemaTimestamp < $localSchemaTimestamp) { + $this->logger->info("No update is needed"); + return; + } + + $this->logger->info("Update needed with timestamp $remoteSchemaTimestamp"); + + $installSchemaCallback = function() use($remoteSchemaTimestamp, $lock) { + $newSchemaPath = $this->updateSchemaFromSharedResource($remoteSchemaTimestamp); + if ($this->logger) { + $this->logger->info("Applying the updated schema ($newSchemaPath -> $this->definitionsDirectory)"); + } + $fs = new FileSystem(); + $fs->remove($this->definitionsDirectory); + $fs->rename($newSchemaPath, $this->definitionsDirectory); + $lock->release(); + }; + register_shutdown_function($installSchemaCallback); + } + + /** + * @return string the path to the new schema + */ + private function updateSchemaFromSharedResource(int $timestamp): string + { + $updatePath = $this->definitionsDirectory . '_' . time(); + $this->logger->info("Synchronizing files from shared schema to $updatePath"); + + $fs = new FileSystem(); + foreach ($this->sharedSchema->getFiles($timestamp) as $name => $contents) { + $fs->dumpFile("$updatePath/$name", $contents); + } + + return $updatePath; + } +}