feat: SSL (MeltanoLabs#162)

Implements SSL for postgres connections, as in
MeltanoLabs/target-postgres#131.

Closes MeltanoLabs#117

---------

Co-authored-by: Derek Visch <[email protected]>

.github/workflows/test.yml

@@ -5,6 +5,8 @@ on:
     types: [opened, synchronize, reopened]
   push:
     branches: [main]
+  workflow_dispatch:
+    inputs: {}
 
 jobs:
   tests:
@@ -34,6 +36,13 @@ jobs:
     - name: Fix Permissions
       run: |
         chmod 777 -R ./ssh_tunnel/ssh-server-config
+        chmod 600 ssl/server.key
+        sudo chown 999:999 ssl/server.key
+        chmod 600 ssl/pkey.key
+
+    - name: Set up Postgres container
+      run: |
+        docker compose -f docker-compose.yml up -d
 
     - uses: isbang/[email protected]
 

README.md

@@ -15,16 +15,28 @@ Built with the [Meltano Singer SDK](https://sdk.meltano.com).
 
 ## Settings
 
-| Setting             | Required | Default | Description |
-|:--------------------|:--------:|:-------:|:------------|
-| sqlalchemy_url      | True     | None    | Example postgresql://postgres:postgres@localhost:5432/postgres |
-| ssh_tunnel          | False    | None    | SSH Tunnel Configuration, this is a json object. |
+| Setting                      | Required | Default | Description |
+|:-----------------------------|:--------:|:-------:|:------------|
+| host                         | False    | None    | Hostname for postgres instance. Note if sqlalchemy_url is set this will be ignored. |
+| port                         | False    |    5432 | The port on which postgres is awaiting connection. Note if sqlalchemy_url is set this will be ignored. |
+| user                         | False    | None    | User name used to authenticate. Note if sqlalchemy_url is set this will be ignored. |
+| password                     | False    | None    | Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. |
+| database                     | False    | None    | Database name. Note if sqlalchemy_url is set this will be ignored. |
+| sqlalchemy_url               | False    | None    | Example postgresql://[username]:[password]@localhost:5432/[db_name] |
+| ssh_tunnel                   | False    | None    | SSH Tunnel Configuration, this is a json object |
 | ssh_tunnel.enable   | True (if ssh_tunnel set) | False   | Enable an ssh tunnel (also known as bastion host), see the other ssh_tunnel.* properties for more details.
 | ssh_tunnel.host | True (if ssh_tunnel set) | False   | Host of the bastion host, this is the host we'll connect to via ssh
 | ssh_tunnel.username | True (if ssh_tunnel set) | False   |Username to connect to bastion host
 | ssh_tunnel.port | True (if ssh_tunnel set) | 22 | Port to connect to bastion host
 | ssh_tunnel.private_key | True (if ssh_tunnel set) | None | Private Key for authentication to the bastion host
 | ssh_tunnel.private_key_password | False | None | Private Key Password, leave None if no password is set
+| ssl_enable                   | False    |       0 | Whether or not to use ssl to verify the server's identity. Use ssl_certificate_authority and ssl_mode for further customization. To use a client certificate to authenticate yourself to the server, use ssl_client_certificate_enable instead. Note if sqlalchemy_url is set this will be ignored. |
+| ssl_client_certificate_enable| False    |       0 | Whether or not to provide client-side certificates as a method of authentication to the server. Use ssl_client_certificate and ssl_client_private_key for further customization. To use SSL to verify the server's identity, use ssl_enable instead. Note if sqlalchemy_url is set this will be ignored. |
+| ssl_mode                     | False    | verify-full | SSL Protection method, see [postgres documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) for more information. Must be one of disable, allow, prefer, require, verify-ca, or verify-full. Note if sqlalchemy_url is set this will be ignored. |
+| ssl_certificate_authority    | False    | ~/.postgresql/root.crl | The certificate authority that should be used to verify the server's identity. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. Note if sqlalchemy_url is set this will be ignored. |
+| ssl_client_certificate       | False    | ~/.postgresql/postgresql.crt | The certificate that should be used to verify your identity to the server. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. Note if sqlalchemy_url is set this will be ignored. |
+| ssl_client_private_key       | False    | ~/.postgresql/postgresql.key | The private key for the certificate you provided. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. Note if sqlalchemy_url is set this will be ignored. |
+| ssl_storage_directory        | False    | .secrets | The folder in which to store SSL certificates provided as raw values. When a certificate/key is provided as a raw value instead of as a filepath, it must be written to a file before it can be used. This configuration option determines where that file is created. |
 | stream_maps         | False    | None    | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
 | stream_map_config   | False    | None    | User-defined config values to be used within map expressions. |
 | flattening_enabled  | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |

docker-compose.yml

@@ -30,6 +30,24 @@ services:
       inner:
         ipv4_address: 10.5.0.5
 
+  postgres_ssl:
+    image: postgres:latest
+    command: postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key -c ssl_ca_file=/var/lib/postgresql/ca.crt -c hba_file=/var/lib/postgresql/pg_hba.conf
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_HOST_AUTH_METHOD: cert clientcert=verify-full
+      POSTGRES_INITDB_ARGS: --auth-host=cert
+    # Not placed in the data directory (/var/lib/postgresql/data) because of https://gist.github.com/mrw34/c97bb03ea1054afb551886ffc8b63c3b?permalink_comment_id=2678568#gistcomment-2678568
+    volumes:
+      - ./ssl/server.crt:/var/lib/postgresql/server.crt # Certificate verifying the server's identity to the client.
+      - ./ssl/server.key:/var/lib/postgresql/server.key # Private key to verify the server's certificate is legitimate.
+      - ./ssl/ca.crt:/var/lib/postgresql/ca.crt # Certificate authority to use when verifying the client's identity to the server.
+      - ./ssl/pg_hba.conf:/var/lib/postgresql/pg_hba.conf # Configuration file to allow connection over SSL.
+    ports:
+      - "5433:5432"
+
 networks:
   inner:
     driver: bridge

meltano.yml

@@ -25,6 +25,17 @@ plugins:
     - name: ssh_tunnel.host
     - name: ssh_tunnel.username
     - name: ssh_tunnel.port
+    - name: ssl_enable
+      kind: boolean
+    - name: ssl_client_certificate_enable
+      kind: boolean
+    - name: ssl_mode
+    - name: ssl_certificate_authority
+      kind: password
+    - name: ssl_client_certificate
+      kind: password
+    - name: ssl_client_private_key
+      kind: password
     select:
     - "*.*"
   loaders: