Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CT-1687 update odbc #170

Merged
merged 5 commits into from
Sep 23, 2024
Merged

CT-1687 update odbc #170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
596f1b9
CT-1687 update odbc driver
jirkasemmler Sep 18, 2024
e45f619
update php
jirkasemmler Sep 18, 2024
34413ad
Drop TD support from image
jirkasemmler Sep 18, 2024
79ba0e3
Drop TD support from image
jirkasemmler Sep 18, 2024
fbc6bc1
Fix stan
jirkasemmler Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 2 additions & 28 deletions packages/php-db-import-export/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
ARG PHP_VERSION=8.1.27
ARG PHP_VERSION=8.2.20

FROM quay.io/keboola/aws-cli as td
ARG AWS_SECRET_ACCESS_KEY
ARG AWS_ACCESS_KEY_ID
RUN /usr/bin/aws s3 cp s3://keboola-drivers/teradata/tdodbc1710-17.10.00.08-1.x86_64.deb /tmp/teradata/tdodbc.deb
RUN /usr/bin/aws s3 cp s3://keboola-drivers/teradata/utils/TeradataToolsAndUtilitiesBase__ubuntu_x8664.17.00.34.00.tar.gz /tmp/teradata/tdutils.tar.gz
RUN /usr/bin/aws s3 cp s3://keboola-drivers/exasol/EXASOL_ODBC-7.1.10.tar.gz /tmp/exasol/odbc.tar.gz

FROM php:${PHP_VERSION:-8.1}-cli-bullseye
Expand All @@ -17,7 +15,7 @@ ENV COMPOSER_ALLOW_SUPERUSER 1
ENV COMPOSER_PROCESS_TIMEOUT 7200

ARG SQLSRV_VERSION=5.10.1
ARG SNOWFLAKE_ODBC_VERSION=2.25.12
ARG SNOWFLAKE_ODBC_VERSION=3.4.1
ARG SNOWFLAKE_GPG_KEY=630D9F3CAB551AF3
ARG GOOGLE_CLOUD_CLI_VERSION=393.0.0

Expand Down Expand Up @@ -106,30 +104,6 @@ RUN set -ex; \
echo "\n[exasol]\nDriver=/opt/exasol/libexaodbc-uo2214lv2.so\n" >> /etc/odbcinst.ini;\
rm -rf /tmp/exasol;

# Teradata ODBC
COPY --from=td /tmp/teradata/tdodbc.deb /tmp/teradata/tdodbc.deb
COPY docker/teradata/odbc.ini /tmp/teradata/odbc_td.ini
COPY docker/teradata/odbcinst.ini /tmp/teradata/odbcinst_td.ini

RUN dpkg -i /tmp/teradata/tdodbc.deb \
&& cat /tmp/teradata/odbc_td.ini >> /etc/odbc.ini \
&& cat /tmp/teradata/odbcinst_td.ini >> /etc/odbcinst.ini \
&& rm -r /tmp/teradata \
&& docker-php-ext-configure pdo_odbc --with-pdo-odbc=unixODBC,/usr \
&& docker-php-ext-install pdo_odbc

ENV ODBCHOME = /opt/teradata/client/ODBC_64/
ENV ODBCINI = /opt/teradata/client/ODBC_64/odbc.ini
ENV ODBCINST = /opt/teradata/client/ODBC_64/odbcinst.ini
ENV LD_LIBRARY_PATH = /opt/teradata/client/ODBC_64/lib

# Teradata Utils
COPY --from=td /tmp/teradata/tdutils.tar.gz /tmp/teradata/tdutils.tar.gz
RUN cd /tmp/teradata \
&& tar -xvaf tdutils.tar.gz \
&& sh /tmp/teradata/TeradataToolsAndUtilitiesBase/.setup.sh tptbase s3axsmod azureaxsmod \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/teradata

#php odbc
RUN docker-php-ext-configure pdo_odbc --with-pdo-odbc=unixODBC,/usr \
Expand Down
4 changes: 0 additions & 4 deletions packages/php-db-import-export/composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,13 @@
"tests-synapse": "SUITE=tests-synapse-sas STORAGE_TYPE=ABS CREDENTIALS_IMPORT_TYPE=SAS CREDENTIALS_EXPORT_TYPE=MASTER_KEY TABLE_DISTRIBUTION=HASH phpunit --colors=always --testsuite tests-synapse",
"tests-synapse-mi": "SUITE=tests-synapse-mi STORAGE_TYPE=ABS CREDENTIALS_IMPORT_TYPE=MANAGED_IDENTITY CREDENTIALS_EXPORT_TYPE=MANAGED_IDENTITY phpunit --colors=always --testsuite synapse-mi",
"tests-exasol": "SUITE=tests-exasol STORAGE_TYPE=S3 phpunit --colors=always --testsuite exasol",
"tests-teradata-tpt-s3": "SUITE=tests-teradata-tpt-s3 TERADATA_CSV_ADAPTER=TPT STORAGE_TYPE=S3 phpunit --colors=always --debug --testsuite tests-teradata-s3",
"tests-teradata-tpt-abs": "SUITE=tests-teradata-tpt-abs TERADATA_CSV_ADAPTER=TPT STORAGE_TYPE=ABS phpunit --colors=always --debug --testsuite tests-teradata-abs",
"tests-bigquery": "SUITE=tests-bigquery STORAGE_TYPE=GCS phpunit --colors=always --testsuite tests-bigquery",
"tests-functional": [
"@tests-snowflake-abs",
"@tests-snowflake-s3",
"@tests-synapse",
"@tests-synapse-mi",
"@tests-exasol",
"@tests-teradata-tpt-s3",
"@tests-teradata-tpt-abs",
"@tests-bigquery"
],
"tests": [
Expand Down
4 changes: 2 additions & 2 deletions packages/php-db-import-export/phpstan-baseline.neon
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ parameters:

-
message: "#^Cannot access offset 'entries' on mixed\\.$#"
count: 3
count: 2
path: src/Storage/ABS/SourceFile.php

-
Expand Down Expand Up @@ -333,4 +333,4 @@ parameters:
-
message: "#^Variable \\$destination in PHPDoc tag @var does not match assigned variable \\$options\\.$#"
count: 1
path: tests/unit/Storage/ABS/SynapseExportAdapterTest.php
path: tests/unit/Storage/ABS/SynapseExportAdapterTest.php
8 changes: 0 additions & 8 deletions packages/php-db-import-export/phpunit.xml.dist
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,6 @@
<exclude>tests/functional/Synapse/FromTableCTASAdapterSqlBuilderTest.php</exclude>
</testsuite>

<testsuite name="tests-teradata-s3">
<directory>tests/functional/Teradata</directory>
</testsuite>

<testsuite name="tests-teradata-abs">
<directory>tests/functional/Teradata</directory>
</testsuite>

<testsuite name="tests-bigquery">
<directory>tests/functional/Bigquery</directory>
</testsuite>
Expand Down
6 changes: 3 additions & 3 deletions packages/php-db-import-export/src/Backend/Exasol/ExasolException.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public static function covertException(DBALAlias $e): Throwable
// strip query from message, there are things like SAS tokens and internal table names
preg_match('/^(An exception occurred.*?)(Following error.*)=?.\(Session.*/ms', $e->getMessage(), $matches);

return FileNotFoundException::createFromDbalException($e, $matches[2]);
return FileNotFoundException::createFromDbalException($e, $matches[2] ?? '');
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to je asi zvysenim phpka a citlivosti stanu

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

jj on rozumí teď regexom :)

}

if (strpos($e->getMessage(), self::INVALID_CSV_DATA) !== false) {
Expand All @@ -37,7 +37,7 @@ public static function covertException(DBALAlias $e): Throwable
$matches,
);

return InvalidSourceDataException::createFromDbalException($e, $matches[2]);
return InvalidSourceDataException::createFromDbalException($e, $matches[2] ?? '');
}

if (strpos($e->getMessage(), self::CONSTRAINT_VIOLATION_NOT_NULL) !== false) {
Expand All @@ -49,7 +49,7 @@ public static function covertException(DBALAlias $e): Throwable
);

return new Exception(
'Load error: ' . ucfirst($matches[2]) . ').',
'Load error: ' . ucfirst(($matches[2] ?? '')) . ').',
Exception::UNKNOWN_ERROR,
);
}
Expand Down
23 changes: 2 additions & 21 deletions packages/php-table-backend-utils/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
FROM quay.io/keboola/aws-cli
ARG AWS_SECRET_ACCESS_KEY
ARG AWS_ACCESS_KEY_ID
RUN /usr/bin/aws s3 cp s3://keboola-drivers/teradata/tdodbc1710-17.10.00.08-1.x86_64.deb /tmp/teradata/tdodbc.deb
RUN /usr/bin/aws s3 cp s3://keboola-drivers/exasol/EXASOL_ODBC-7.1.10.tar.gz /tmp/exasol/odbc.tar.gz

FROM php:8.1.27-cli-bullseye
FROM php:8.2.20-cli-bullseye
MAINTAINER Keboola <[email protected]>

ARG COMPOSER_FLAGS="--prefer-dist --no-interaction"
Expand All @@ -13,7 +12,7 @@ ENV COMPOSER_ALLOW_SUPERUSER 1
ENV COMPOSER_PROCESS_TIMEOUT 3600

ARG SQLSRV_VERSION=5.10.1
ARG SNOWFLAKE_ODBC_VERSION=2.25.12
ARG SNOWFLAKE_ODBC_VERSION=3.4.1
ARG SNOWFLAKE_GPG_KEY=630D9F3CAB551AF3

WORKDIR /code/
Expand Down Expand Up @@ -88,24 +87,6 @@ RUN mkdir -p ~/.gnupg \
&& gpg --batch --delete-key --yes $SNOWFLAKE_GPG_KEY \
&& dpkg -i /tmp/snowflake-odbc.deb

# Teradata
COPY --from=0 /tmp/teradata/tdodbc.deb /tmp/teradata/tdodbc.deb
COPY docker/teradata/odbc.ini /tmp/teradata/odbc_td.ini
COPY docker/teradata/odbcinst.ini /tmp/teradata/odbcinst_td.ini

RUN dpkg -i /tmp/teradata/tdodbc.deb \
&& cat /tmp/teradata/odbc_td.ini >> /etc/odbc.ini \
&& cat /tmp/teradata/odbcinst_td.ini >> /etc/odbcinst.ini \
&& rm -r /tmp/teradata \
&& docker-php-ext-configure pdo_odbc --with-pdo-odbc=unixODBC,/usr \
&& docker-php-ext-install pdo_odbc \
&& docker-php-source delete

ENV ODBCHOME = /opt/teradata/client/ODBC_64/
ENV ODBCINI = /opt/teradata/client/ODBC_64/odbc.ini
ENV ODBCINST = /opt/teradata/client/ODBC_64/odbcinst.ini
ENV LD_LIBRARY_PATH = /opt/teradata/client/ODBC_64/lib

#Exasol
COPY --from=0 /tmp/exasol/odbc.tar.gz /tmp/exasol/odbc.tar.gz
RUN set -ex; \
Expand Down
2 changes: 0 additions & 2 deletions packages/php-table-backend-utils/composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,11 @@
"tests-exasol": "phpunit --colors=always --testsuite Exasol",
"tests-snowflake": "phpunit --colors=always --testsuite Snowflake",
"tests-synapse": "phpunit --colors=always --testsuite Synapse",
"tests-teradata": "phpunit --colors=always --testsuite Teradata",
"tests-bigquery": "phpunit --colors=always --testsuite Bigquery",
"tests-functional": [
"@tests-exasol",
"@tests-snowflake",
"@tests-synapse",
"@tests-teradata",
"@tests-bigquery"
],
"tests": [
Expand Down
3 changes: 0 additions & 3 deletions packages/php-table-backend-utils/phpunit.xml.dist
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
<testsuite name="Synapse">
<directory>tests/Functional/Synapse</directory>
</testsuite>
<testsuite name="Teradata">
<directory>tests/Functional/Teradata</directory>
</testsuite>
<testsuite name="Bigquery">
<directory>tests/Functional/Bigquery</directory>
</testsuite>
Expand Down
4 changes: 2 additions & 2 deletions packages/php-table-backend-utils/src/Column/Exasol/ExasolColumn.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,12 @@ private static function extractColumnLength(array $colData): string
$colType = $colData['COLUMN_TYPE'];
if ($colData['TYPE_NAME'] === Exasol::TYPE_INTERVAL_DAY_TO_SECOND) {
preg_match('/INTERVAL DAY\((?P<valDays>\d)\) TO SECOND\((?P<valSeconds>\d)\)/', $colType, $output_array);
return $output_array['valDays'] . ',' . $output_array['valSeconds'];
return ($output_array['valDays'] ?? '') . ',' . ($output_array['valSeconds'] ?? '');
}

if ($colData['TYPE_NAME'] === Exasol::TYPE_INTERVAL_YEAR_TO_MONTH) {
preg_match('/INTERVAL YEAR\((?P<val>\d)\) TO MONTH/', $colType, $output_array);
return $output_array['val'];
return $output_array['val'] ?? '';
}

$precision = $colData['COLUMN_NUM_PREC'];
Expand Down
Loading