Fix MSSQL tests

MobileTeleSystems · Jul 29, 2024 · 267cc50 · 267cc50
1 parent df869d0
commit 267cc50
Show file tree

Hide file tree

Showing 17 changed files with 222 additions and 7 deletions.
diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml
@@ -95,3 +95,17 @@ jobs:
         with:
           name: coverage-clickhouse-${{ inputs.clickhouse-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
           path: reports/*
+
+      - name: Dump Clickhouse logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: ${{ inputs.clickhouse-image }}
+          dest: ./logs
+
+      - name: Upload Clickhouse logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-clickhouse-${{ inputs.clickhouse-version }}-spark-${{ inputs.spark-version }}
+          path: logs/*
diff --git a/.github/workflows/test-ftp.yml b/.github/workflows/test-ftp.yml
@@ -67,6 +67,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m ftp
 
+      - name: Dump FTP logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: chonjay21/ftps
+          dest: ./logs
+
+      - name: Upload FTP logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-ftp-${{ inputs.ftp-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Shutdown FTP
         if: always()
         run: |

diff --git a/.github/workflows/test-ftps.yml b/.github/workflows/test-ftps.yml
@@ -67,6 +67,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m ftps
 
+      - name: Dump FTPS logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: chonjay21/ftps
+          dest: ./logs
+
+      - name: Upload FTPS logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-ftps-${{ inputs.ftps-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Shutdown FTPS
         if: always()
         run: |

diff --git a/.github/workflows/test-greenplum.yml b/.github/workflows/test-greenplum.yml
@@ -118,6 +118,20 @@ jobs:
           GREENPLUM_PACKAGES_USER: ${{ secrets.GREENPLUM_PACKAGES_USER }}
           GREENPLUM_PACKAGES_PASSWORD: ${{ secrets.GREENPLUM_PACKAGES_PASSWORD }}
 
+      - name: Dump Greenplum logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: andruche/greenplum
+          dest: ./logs
+
+      - name: Upload Greenplum logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-greenplum-${{ inputs.greenplum-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-hdfs.yml b/.github/workflows/test-hdfs.yml
@@ -90,6 +90,20 @@ jobs:
           echo "127.0.0.1 hdfs" | sudo tee -a /etc/hosts
           ./pytest_runner.sh -m hdfs
 
+      - name: Dump HDFS logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mtsrus/hadoop
+          dest: ./logs
+
+      - name: Upload HDFS logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-hdfs-${{ inputs.hadoop-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Shutdown HDFS
         if: always()
         run: |

diff --git a/.github/workflows/test-kafka.yml b/.github/workflows/test-kafka.yml
@@ -116,6 +116,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m kafka
 
+      - name: Dump Kafka logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: bitnami/kafka
+          dest: ./logs
+
+      - name: Upload Kafka logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-kafka-${{ inputs.kafka-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-mongodb.yml b/.github/workflows/test-mongodb.yml
@@ -88,6 +88,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m mongodb
 
+      - name: Dump MongoDB logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mongo
+          dest: ./logs
+
+      - name: Upload MongoDB logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-mongodb-${{ inputs.mongodb-version }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-mssql.yml b/.github/workflows/test-mssql.yml
@@ -88,6 +88,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m mssql
 
+      - name: Dump MSSQL logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mcr.microsoft.com/mssql/server
+          dest: ./logs
+
+      - name: Upload MSSQL logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-mssql-${{ inputs.mssql-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Shutdown MSSQL
         if: always()
         run: |

diff --git a/.github/workflows/test-mysql.yml b/.github/workflows/test-mysql.yml
@@ -90,6 +90,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m mysql
 
+      - name: Dump MySQL logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mysql
+          dest: ./logs
+
+      - name: Upload MySQL logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-mysql-${{ inputs.mysql-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-oracle.yml b/.github/workflows/test-oracle.yml
@@ -38,7 +38,7 @@ jobs:
     runs-on: ${{ inputs.os }}
     services:
       oracle:
-        image: "${{ inputs.oracle-image }}:${{ inputs.oracle-version }}"
+        image: ${{ inputs.oracle-image }}:${{ inputs.oracle-version }}
         env:
           TZ: UTC
           ORACLE_PASSWORD: maaxohmiGe9eep5x
@@ -110,6 +110,20 @@ jobs:
           export "ONETL_ORA_SERVICE_NAME=${{ inputs.db-name }}"
           ./pytest_runner.sh -m oracle
 
+      - name: Dump Oracle logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: ${{ inputs.oracle-image }}
+          dest: ./logs
+
+      - name: Upload Oracle logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-oracle-${{ inputs.oracle-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-postgres.yml b/.github/workflows/test-postgres.yml
@@ -89,6 +89,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m postgres
 
+      - name: Dump Postgres logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: postgres
+          dest: ./logs
+
+      - name: Upload Postgres logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-postgres-${{ inputs.postgres-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-s3.yml b/.github/workflows/test-s3.yml
@@ -90,6 +90,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m s3
 
+      - name: Dump S3 logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: bitnami/minio
+          dest: ./logs
+
+      - name: Upload S3 logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-s3-${{ inputs.minio-version }}-spark-${{ inputs.spark-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-samba.yml b/.github/workflows/test-samba.yml
@@ -65,6 +65,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m samba
 
+      - name: Dump Samba logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: elswork/samba
+          dest: ./logs
+
+      - name: Upload Samba logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-samba-${{ inputs.server-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Shutdown Samba
         if: always()
         run: |

diff --git a/.github/workflows/test-sftp.yml b/.github/workflows/test-sftp.yml
@@ -67,6 +67,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m sftp
 
+      - name: Dump SFTP logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: linuxserver/openssh-server
+          dest: ./logs
+
+      - name: Upload SFTP logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-sftp-${{ inputs.openssh-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/test-webdav.yml b/.github/workflows/test-webdav.yml
@@ -67,6 +67,20 @@ jobs:
           source ./env
           ./pytest_runner.sh -m webdav
 
+      - name: Dump WebDAV logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: chonjay21/webdav
+          dest: ./logs
+
+      - name: Upload WebDAV logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: container-logs-webdav-${{ inputs.webdav-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+          path: logs/*
+
       - name: Shutdown WebDAV
         if: always()
         run: |

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -91,9 +91,6 @@ services:
     image: ${MSSQL_IMAGE:-mcr.microsoft.com/mssql/server:latest}
     restart: unless-stopped
     env_file: .env.dependencies
-    environment:
-      # fix for https://github.com/microsoft/mssql-docker/issues/892
-      PATH: /opt/mssql-tools18/bin:/opt/mssql-tools/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
     ports:
       - 1433:1433
     volumes:
@@ -104,7 +101,8 @@ services:
     platform: linux/amd64
     healthcheck:
       # Container is healthy only when database is created
-      test: ["CMD-SHELL", "sqlcmd -S localhost -d $$MSSQL_DATABASE -U $$MSSQL_USER -P $$MSSQL_PASSWORD -Q 'SELECT 1'"]
+      # TODO: replace with SELECT after fixing sqlcmd location: https://github.com/microsoft/mssql-docker/issues/892
+      test: ["CMD-SHELL", "ls -lsah /var/opt/mssql/data/onetl.mdf"]
       interval: 10s
       timeout: 5s
       retries: 5

diff --git a/docker/mssql/configure-db.sh b/docker/mssql/configure-db.sh
@@ -12,14 +12,21 @@ TIMEOUT=60
 START=$(date +%s)
 echo "Configure DB script started at $(date)"
 
+# fix for https://github.com/microsoft/mssql-docker/issues/892
+if [[ -d "/opt/mssql-tools18/bin" ]]; then
+    SQLCMD="/opt/mssql-tools18/bin/sqlcmd -No"
+else
+    SQLCMD=/opt/mssql-tools/bin/sqlcmd
+fi
+
 while true; do
     DELTA=$(($(date +%s) - START))
     if [[ $DELTA -gt $TIMEOUT ]]; then
         echo "ERROR: SQL Server took more than ${TIMEOUT} seconds to START up or one or more databases are not in an ONLINE state"
         exit 1
     fi
 
-    DBSTATUS=$(sqlcmd -h -1 -t 1 -U sa -P ${MSSQL_SA_PASSWORD} -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases" 2>/dev/null | sed -e 's/^[[:space:]]*//')
+    DBSTATUS=$($SQLCMD -h -1 -t 1 -U sa -P ${MSSQL_SA_PASSWORD} -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases" 2>/dev/null | sed -e 's/^[[:space:]]*//')
     ERRCODE=$?
     if [[ "$DBSTATUS" -eq "0" && "$ERRCODE" -eq "0" ]]; then
         echo "INFO: Database ready."
@@ -32,5 +39,5 @@ done
 
 # Run the setup script to create the DB and the schema in the DB
 echo "Running setup.sql";
-sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql;
+$SQLCMD -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/config/setup.sql;
 echo "Success";