-
Notifications
You must be signed in to change notification settings - Fork 60
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feature] Add SQL Data Source (#379)
* Add SQL data source with connections select * Create sql data loader * Add sql format to file format selection * Update docs * Apply php-cs-fixer changes * Add doctrine dependencies * Replace executeQuery with execute to match doctrine dependency in SqlLoader * Add missing strict_types declaration in php files --------- Co-authored-by: msoroka <[email protected]>
- Loading branch information
Showing
12 changed files
with
411 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| /** | ||
| * Pimcore | ||
| * | ||
| * This source file is available under two different licenses: | ||
| * - GNU General Public License version 3 (GPLv3) | ||
| * - Pimcore Commercial License (PCL) | ||
| * Full copyright and license information is available in | ||
| * LICENSE.md which is distributed with this source code. | ||
| * | ||
| * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.org) | ||
| * @license http://www.pimcore.org/license GPLv3 and PCL | ||
| */ | ||
|
|
||
| namespace Pimcore\Bundle\DataImporterBundle\Controller; | ||
|
|
||
| use Exception; | ||
| use Pimcore\Controller\UserAwareController; | ||
| use Symfony\Component\HttpFoundation\JsonResponse; | ||
| use Symfony\Component\Routing\Annotation\Route; | ||
|
|
||
| /** | ||
| * @Route("/admin/pimcoredataimporter/") | ||
| */ | ||
| class ConnectionController extends UserAwareController | ||
| { | ||
| /** | ||
| * @Route("connections", name="pimcore_dataimporter_connections", methods={"GET"}) | ||
| * | ||
| * @throws Exception | ||
| */ | ||
| public function connectionAction(): JsonResponse | ||
| { | ||
| $connections = $this->getParameter('doctrine.connections'); | ||
|
|
||
| if (!is_array($connections)) { | ||
| throw new Exception('Doctrine connection not returned as array'); | ||
| } | ||
|
|
||
| $mappedConnections = array_map(fn ($key, $value): array => [ | ||
| 'name' => $key, | ||
| 'value' => $value | ||
| ], array_keys($connections), $connections); | ||
|
|
||
| return $this->json($mappedConnections); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| /** | ||
| * Pimcore | ||
| * | ||
| * This source file is available under two different licenses: | ||
| * - GNU General Public License version 3 (GPLv3) | ||
| * - Pimcore Commercial License (PCL) | ||
| * Full copyright and license information is available in | ||
| * LICENSE.md which is distributed with this source code. | ||
| * | ||
| * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.org) | ||
| * @license http://www.pimcore.org/license GPLv3 and PCL | ||
| */ | ||
|
|
||
| namespace Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter; | ||
|
|
||
| class SqlFileInterpreter extends JsonFileInterpreter | ||
| { | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| /** | ||
| * Pimcore | ||
| * | ||
| * This source file is available under two different licenses: | ||
| * - GNU General Public License version 3 (GPLv3) | ||
| * - Pimcore Commercial License (PCL) | ||
| * Full copyright and license information is available in | ||
| * LICENSE.md which is distributed with this source code. | ||
| * | ||
| * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.org) | ||
| * @license http://www.pimcore.org/license GPLv3 and PCL | ||
| */ | ||
|
|
||
| namespace Pimcore\Bundle\DataImporterBundle\DataSource\Loader; | ||
|
|
||
| use Doctrine\DBAL\Connection; | ||
| use Doctrine\DBAL\Exception; | ||
| use League\Flysystem\Filesystem; | ||
| use League\Flysystem\FilesystemException; | ||
| use League\Flysystem\Local\LocalFilesystemAdapter; | ||
| use Pimcore; | ||
| use Pimcore\Bundle\DataImporterBundle\Exception\InvalidConfigurationException; | ||
| use Symfony\Component; | ||
|
|
||
| class SqlLoader implements DataLoaderInterface | ||
| { | ||
| private string $connection; | ||
| private string $select; | ||
| private string $from; | ||
| private string $where; | ||
| private string $groupBy; | ||
|
|
||
| private string $importFilePath; | ||
| private Connection $databaseConnection; | ||
|
|
||
| public function __construct(private Component\Filesystem\Filesystem $filesystem) | ||
| { | ||
| } | ||
|
|
||
| /** | ||
| * @throws InvalidConfigurationException | ||
| * @throws Exception | ||
| * @throws FilesystemException | ||
| */ | ||
| public function loadData(): string | ||
| { | ||
| $this->setUpConnection(); | ||
| $this->setUpImportFilePath(); | ||
|
|
||
| $queryBuilder = $this->databaseConnection->createQueryBuilder(); | ||
| $queryBuilder->select($this->select) | ||
| ->from($this->from); | ||
|
|
||
| if (!empty($this->where)) { | ||
| $queryBuilder->where($this->where); | ||
| } | ||
|
|
||
| if (!empty($this->groupBy)) { | ||
| $queryBuilder->groupBy($this->groupBy); | ||
| } | ||
|
|
||
| $result = $queryBuilder->execute()->fetchAllAssociative(); | ||
|
|
||
| $filesystemLocal = new Filesystem(new LocalFilesystemAdapter('/')); | ||
| $stream = fopen('php://temp', 'r+'); | ||
| $resultAsJson = json_encode($result); | ||
|
|
||
| fwrite($stream, $resultAsJson); | ||
| rewind($stream); | ||
|
|
||
| $filesystemLocal->writeStream($this->importFilePath, $stream); | ||
|
|
||
| return $this->importFilePath; | ||
| } | ||
|
|
||
| public function cleanup(): void | ||
| { | ||
| $this->databaseConnection->close(); | ||
|
|
||
| unlink($this->importFilePath); | ||
| } | ||
|
|
||
| /** | ||
| * @throws InvalidConfigurationException | ||
| */ | ||
| public function setSettings(array $settings): void | ||
| { | ||
| if (empty($settings['connection'])) { | ||
| throw new InvalidConfigurationException('Empty connection.'); | ||
| } | ||
| $this->connection = $settings['connection']; | ||
|
|
||
| if (empty($settings['select'])) { | ||
| throw new InvalidConfigurationException('Empty select.'); | ||
| } | ||
| $this->select = $settings['select']; | ||
|
|
||
| if (empty($settings['from'])) { | ||
| throw new InvalidConfigurationException('Empty from.'); | ||
| } | ||
| $this->from = $settings['from']; | ||
|
|
||
| $this->where = $settings['where']; | ||
| $this->groupBy = $settings['groupBy']; | ||
| } | ||
|
|
||
| /** | ||
| * @throws InvalidConfigurationException | ||
| */ | ||
| private function setUpConnection(): void | ||
| { | ||
| $container = Pimcore::getContainer(); | ||
| $databaseConnection = null; | ||
|
|
||
| if ($container instanceof Component\DependencyInjection\ContainerInterface) { | ||
| $databaseConnection = $container->get($this->connection); | ||
| } | ||
|
|
||
| if (!$databaseConnection instanceof Connection) { | ||
| throw new InvalidConfigurationException('Connection not found.'); | ||
| } | ||
|
|
||
| $this->databaseConnection = $databaseConnection; | ||
| } | ||
|
|
||
| private function setUpImportFilePath(): void | ||
| { | ||
| $folder = PIMCORE_PRIVATE_VAR . '/tmp/datahub/dataimporter/sql-loader/'; | ||
| $this->filesystem->mkdir($folder, 0775); | ||
|
|
||
| $this->importFilePath = $folder . uniqid('sql-import-'); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
src/Resources/public/js/pimcore/configuration/components/interpreter/sql.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| /** | ||
| * Pimcore | ||
| * | ||
| * This source file is available under two different licenses: | ||
| * - GNU General Public License version 3 (GPLv3) | ||
| * - Pimcore Commercial License (PCL) | ||
| * Full copyright and license information is available in | ||
| * LICENSE.md which is distributed with this source code. | ||
| * | ||
| * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.org) | ||
| * @license http://www.pimcore.org/license GPLv3 and PCL | ||
| */ | ||
|
|
||
| pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.sql"); | ||
| pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.sql = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { | ||
|
|
||
| type: 'sql', | ||
|
|
||
| buildSettingsForm: function() { | ||
|
|
||
| if(!this.form) { | ||
| this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { | ||
| defaults: { | ||
| labelWidth: 200, | ||
| width: 600 | ||
| }, | ||
| border: false, | ||
| items: [ | ||
| ] | ||
| }); | ||
| } | ||
|
|
||
| return this.form; | ||
| } | ||
|
|
||
| }); |
Oops, something went wrong.