Merge pull request #6 from MeltanoLabs/ssh_tunnel

feat: SSH Tunnel working

.github/workflows/test.yml

@@ -11,14 +11,6 @@ on:
 jobs:
   tests:
     runs-on: ubuntu-latest
-    services:
-      mysqldb:
-        image: mysql
-        env:
-          MYSQL_ROOT_PASSWORD: password
-          MYSQL_DATABASE: melty
-        ports:
-          - 3307:3306
     env:
       PIP_CONSTRAINT: .github/workflows/constraints.txt
     strategy:
@@ -29,7 +21,11 @@ jobs:
     - name: Checkout code
       uses: actions/[email protected]
 
-    - name: Set up MySQL container
+    - name: Fix Permissions
+      run: |
+        chmod 777 -R ./ssh_tunnel/ssh-server-config
+    
+    - name: Set up MySQL containers (SSH Included)
       run: |
         docker compose -f docker-compose.yml up -d
 
@@ -57,3 +53,6 @@ jobs:
     - name: Run lint
       run: |
         poetry run tox -e lint
+
+
+# To test mysql client 1. docker exec -it 012c /bin/bash (ssh server) 2. apk add mysql mysql-client 3. mysql -h 10.5.0.5 -P 3306 -u root -ppassword 

README.md

@@ -24,6 +24,13 @@ pipx install git+https://github.com/MeltanoLabs/tap-mysql.git@main
 | password            | False    | None    | Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. |
 | database            | False    | None    | Database name. Note if sqlalchemy_url is set this will be ignored. |
 | sqlalchemy_url      | False    | None    | Example mysql://[username]:[password]@localhost:3306/[db_name] |
+| ssh_tunnel                   | False    | None    | SSH Tunnel Configuration, this is a json object |
+| ssh_tunnel.enable   | True (if ssh_tunnel set) | False   | Enable an ssh tunnel (also known as bastion host), see the other ssh_tunnel.* properties for more details.
+| ssh_tunnel.host | True (if ssh_tunnel set) | False   | Host of the bastion host, this is the host we'll connect to via ssh
+| ssh_tunnel.username | True (if ssh_tunnel set) | False   |Username to connect to bastion host
+| ssh_tunnel.port | True (if ssh_tunnel set) | 22 | Port to connect to bastion host
+| ssh_tunnel.private_key | True (if ssh_tunnel set) | None | Private Key for authentication to the bastion host
+| ssh_tunnel.private_key_password | False | None | Private Key Password, leave None if no password is set
 | stream_maps         | False    | None    | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
 | stream_map_config   | False    | None    | User-defined config values to be used within map expressions. |
 | flattening_enabled  | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |
@@ -44,6 +51,10 @@ This Singer tap will automatically import any environment variables within the w
 `.env` if the `--config=ENV` is provided, such that config values will be considered if a matching
 environment variable is set either in the terminal context or in the `.env` file.
 
+### SSH Tunnels (Bastion Hosts)
+
+This tap supports connecting to a Postgres database via an SSH tunnel (also known as a bastion host). This is useful if you need to connect to a database that is not publicly accessible. This is the same as using `ssh -L` and `ssh -R`, but this is done inside the tap itself.
+
 ## Usage
 
 You can easily run `tap-mysql` by itself or in a pipeline using [Meltano](https://meltano.com/).

docker-compose.yml

@@ -5,8 +5,43 @@ services:
   mysqldb:
     image: mysql
     restart: always
+    command: --default-authentication-plugin=mysql_native_password
     environment:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_DATABASE: melty
     ports:
       - 3306:3306 
+  mysqldb_ssh:
+    image: mysql
+    restart: always
+    command: --default-authentication-plugin=mysql_native_password
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_DATABASE: melty
+    networks:
+      inner:
+        ipv4_address: 10.5.0.5
+  ssh:
+    image: ghcr.io/linuxserver/openssh-server:9.1_p1-r2-ls106 # This docker repo has added breaking changes a lot in the last month, pinning this.
+    hostname: openssh-server
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - PUBLIC_KEY_FILE=/config/ssh_host_keys/ssh_host_rsa_key.pub
+      - SUDO_ACCESS=true
+      - PASSWORD_ACCESS=false
+      - USER_NAME=melty
+    volumes:
+      - ./ssh_tunnel/ssh-server-config:/config/ssh_host_keys:ro
+    ports:
+      - "127.0.0.1:2223:2222"
+    networks:
+      - inner
+
+networks:
+  inner:
+    driver: bridge
+    ipam:
+     config:
+       - subnet: 10.5.0.0/16
+         gateway: 10.5.0.1

meltano.yml

@@ -25,6 +25,13 @@ plugins:
     - name: database
     - name: sqlalchemy_url
       kind: password
+    - name: ssh_tunnel.private_key
+      kind: password
+    - name: ssh_tunnel.private_key_password
+      kind: password
+    - name: ssh_tunnel.host
+    - name: ssh_tunnel.username
+    - name: ssh_tunnel.port
   loaders:
   - name: target-jsonl
     variant: andyh1203