diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml index ba3bc21d..db05402d 100644 --- a/.github/workflows/test-clickhouse.yml +++ b/.github/workflows/test-clickhouse.yml @@ -95,3 +95,17 @@ jobs: with: name: coverage-clickhouse-${{ inputs.clickhouse-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} path: reports/* + + - name: Dump Clickhouse logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: ${{ inputs.clickhouse-image }} + dest: ./logs + + - name: Upload Clickhouse logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-clickhouse-${{ inputs.clickhouse-version }}-spark-${{ inputs.spark-version }} + path: logs/* diff --git a/.github/workflows/test-ftp.yml b/.github/workflows/test-ftp.yml index dee06115..8e45ec32 100644 --- a/.github/workflows/test-ftp.yml +++ b/.github/workflows/test-ftp.yml @@ -67,6 +67,20 @@ jobs: source ./env ./pytest_runner.sh -m ftp + - name: Dump FTP logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: chonjay21/ftps + dest: ./logs + + - name: Upload FTP logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-ftp-${{ inputs.ftp-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Shutdown FTP if: always() run: | diff --git a/.github/workflows/test-ftps.yml b/.github/workflows/test-ftps.yml index fadb4406..dfe8ffed 100644 --- a/.github/workflows/test-ftps.yml +++ b/.github/workflows/test-ftps.yml @@ -67,6 +67,20 @@ jobs: source ./env ./pytest_runner.sh -m ftps + - name: Dump FTPS logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: chonjay21/ftps + dest: ./logs + + - name: Upload FTPS logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-ftps-${{ inputs.ftps-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Shutdown FTPS if: always() run: | diff --git a/.github/workflows/test-greenplum.yml b/.github/workflows/test-greenplum.yml index 9bc79fdd..5f24f779 100644 --- a/.github/workflows/test-greenplum.yml +++ b/.github/workflows/test-greenplum.yml @@ -118,6 +118,20 @@ jobs: GREENPLUM_PACKAGES_USER: ${{ secrets.GREENPLUM_PACKAGES_USER }} GREENPLUM_PACKAGES_PASSWORD: ${{ secrets.GREENPLUM_PACKAGES_PASSWORD }} + - name: Dump Greenplum logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: andruche/greenplum + dest: ./logs + + - name: Upload Greenplum logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-greenplum-${{ inputs.greenplum-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-hdfs.yml b/.github/workflows/test-hdfs.yml index 41c339dc..e06f01ed 100644 --- a/.github/workflows/test-hdfs.yml +++ b/.github/workflows/test-hdfs.yml @@ -90,6 +90,20 @@ jobs: echo "127.0.0.1 hdfs" | sudo tee -a /etc/hosts ./pytest_runner.sh -m hdfs + - name: Dump HDFS logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: mtsrus/hadoop + dest: ./logs + + - name: Upload HDFS logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-hdfs-${{ inputs.hadoop-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Shutdown HDFS if: always() run: | diff --git a/.github/workflows/test-kafka.yml b/.github/workflows/test-kafka.yml index 31f5e6e1..b1f06552 100644 --- a/.github/workflows/test-kafka.yml +++ b/.github/workflows/test-kafka.yml @@ -116,6 +116,20 @@ jobs: source ./env ./pytest_runner.sh -m kafka + - name: Dump Kafka logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: bitnami/kafka + dest: ./logs + + - name: Upload Kafka logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-kafka-${{ inputs.kafka-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-mongodb.yml b/.github/workflows/test-mongodb.yml index ec9e7fa4..334bbfc1 100644 --- a/.github/workflows/test-mongodb.yml +++ b/.github/workflows/test-mongodb.yml @@ -88,6 +88,20 @@ jobs: source ./env ./pytest_runner.sh -m mongodb + - name: Dump MongoDB logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: mongo + dest: ./logs + + - name: Upload MongoDB logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-mongodb-${{ inputs.mongodb-version }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-mssql.yml b/.github/workflows/test-mssql.yml index 037a2c13..0865492a 100644 --- a/.github/workflows/test-mssql.yml +++ b/.github/workflows/test-mssql.yml @@ -88,6 +88,20 @@ jobs: source ./env ./pytest_runner.sh -m mssql + - name: Dump MSSQL logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: mcr.microsoft.com/mssql/server + dest: ./logs + + - name: Upload MSSQL logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-mssql-${{ inputs.mssql-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Shutdown MSSQL if: always() run: | diff --git a/.github/workflows/test-mysql.yml b/.github/workflows/test-mysql.yml index 97691a6d..e305af6d 100644 --- a/.github/workflows/test-mysql.yml +++ b/.github/workflows/test-mysql.yml @@ -90,6 +90,20 @@ jobs: source ./env ./pytest_runner.sh -m mysql + - name: Dump MySQL logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: mysql + dest: ./logs + + - name: Upload MySQL logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-mysql-${{ inputs.mysql-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-oracle.yml b/.github/workflows/test-oracle.yml index 2ba490c1..38a21daf 100644 --- a/.github/workflows/test-oracle.yml +++ b/.github/workflows/test-oracle.yml @@ -38,7 +38,7 @@ jobs: runs-on: ${{ inputs.os }} services: oracle: - image: "${{ inputs.oracle-image }}:${{ inputs.oracle-version }}" + image: ${{ inputs.oracle-image }}:${{ inputs.oracle-version }} env: TZ: UTC ORACLE_PASSWORD: maaxohmiGe9eep5x @@ -110,6 +110,20 @@ jobs: export "ONETL_ORA_SERVICE_NAME=${{ inputs.db-name }}" ./pytest_runner.sh -m oracle + - name: Dump Oracle logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: ${{ inputs.oracle-image }} + dest: ./logs + + - name: Upload Oracle logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-oracle-${{ inputs.oracle-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-postgres.yml b/.github/workflows/test-postgres.yml index 7a9022b3..68236134 100644 --- a/.github/workflows/test-postgres.yml +++ b/.github/workflows/test-postgres.yml @@ -89,6 +89,20 @@ jobs: source ./env ./pytest_runner.sh -m postgres + - name: Dump Postgres logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: postgres + dest: ./logs + + - name: Upload Postgres logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-postgres-${{ inputs.postgres-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-s3.yml b/.github/workflows/test-s3.yml index 3179002c..1ef595e6 100644 --- a/.github/workflows/test-s3.yml +++ b/.github/workflows/test-s3.yml @@ -90,6 +90,20 @@ jobs: source ./env ./pytest_runner.sh -m s3 + - name: Dump S3 logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: bitnami/minio + dest: ./logs + + - name: Upload S3 logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-s3-${{ inputs.minio-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-samba.yml b/.github/workflows/test-samba.yml index 4a2e30d1..f7b07131 100644 --- a/.github/workflows/test-samba.yml +++ b/.github/workflows/test-samba.yml @@ -65,6 +65,20 @@ jobs: source ./env ./pytest_runner.sh -m samba + - name: Dump Samba logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: elswork/samba + dest: ./logs + + - name: Upload Samba logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-samba-${{ inputs.server-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Shutdown Samba if: always() run: | diff --git a/.github/workflows/test-sftp.yml b/.github/workflows/test-sftp.yml index eaa5e5a4..2ab8de1c 100644 --- a/.github/workflows/test-sftp.yml +++ b/.github/workflows/test-sftp.yml @@ -67,6 +67,20 @@ jobs: source ./env ./pytest_runner.sh -m sftp + - name: Dump SFTP logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: linuxserver/openssh-server + dest: ./logs + + - name: Upload SFTP logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-sftp-${{ inputs.openssh-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Upload coverage results uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-webdav.yml b/.github/workflows/test-webdav.yml index 34a94326..2ce0e4ef 100644 --- a/.github/workflows/test-webdav.yml +++ b/.github/workflows/test-webdav.yml @@ -67,6 +67,20 @@ jobs: source ./env ./pytest_runner.sh -m webdav + - name: Dump WebDAV logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + images: chonjay21/webdav + dest: ./logs + + - name: Upload WebDAV logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: container-logs-webdav-${{ inputs.webdav-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }} + path: logs/* + - name: Shutdown WebDAV if: always() run: | diff --git a/docker-compose.yml b/docker-compose.yml index 41d8af80..f5859bb5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,9 +91,6 @@ services: image: ${MSSQL_IMAGE:-mcr.microsoft.com/mssql/server:latest} restart: unless-stopped env_file: .env.dependencies - environment: - # fix for https://github.com/microsoft/mssql-docker/issues/892 - PATH: /opt/mssql-tools18/bin:/opt/mssql-tools/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ports: - 1433:1433 volumes: @@ -104,7 +101,8 @@ services: platform: linux/amd64 healthcheck: # Container is healthy only when database is created - test: ["CMD-SHELL", "sqlcmd -S localhost -d $$MSSQL_DATABASE -U $$MSSQL_USER -P $$MSSQL_PASSWORD -Q 'SELECT 1'"] + # TODO: replace with SELECT after fixing sqlcmd location: https://github.com/microsoft/mssql-docker/issues/892 + test: ["CMD-SHELL", "ls -lsah /var/opt/mssql/data/onetl.mdf"] interval: 10s timeout: 5s retries: 5 diff --git a/docker/mssql/configure-db.sh b/docker/mssql/configure-db.sh index 38e224e3..6c31f067 100755 --- a/docker/mssql/configure-db.sh +++ b/docker/mssql/configure-db.sh @@ -12,6 +12,13 @@ TIMEOUT=60 START=$(date +%s) echo "Configure DB script started at $(date)" +# fix for https://github.com/microsoft/mssql-docker/issues/892 +if [[ -d "/opt/mssql-tools18/bin" ]]; then + SQLCMD="/opt/mssql-tools18/bin/sqlcmd -No" +else + SQLCMD=/opt/mssql-tools/bin/sqlcmd +fi + while true; do DELTA=$(($(date +%s) - START)) if [[ $DELTA -gt $TIMEOUT ]]; then @@ -19,7 +26,7 @@ while true; do exit 1 fi - DBSTATUS=$(sqlcmd -h -1 -t 1 -U sa -P ${MSSQL_SA_PASSWORD} -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases" 2>/dev/null | sed -e 's/^[[:space:]]*//') + DBSTATUS=$($SQLCMD -h -1 -t 1 -U sa -P ${MSSQL_SA_PASSWORD} -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases" 2>/dev/null | sed -e 's/^[[:space:]]*//') ERRCODE=$? if [[ "$DBSTATUS" -eq "0" && "$ERRCODE" -eq "0" ]]; then echo "INFO: Database ready." @@ -32,5 +39,5 @@ done # Run the setup script to create the DB and the schema in the DB echo "Running setup.sql"; -sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql; +$SQLCMD -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql; echo "Success";