Michelp/zymbit (#16)

* zymbit first draft.

* aws kms getkey example.

* fix bug in aws key generation.

* google kms example.

* add zymkey getkey script.

* add a sleep to suppress anoying connect error on faster computer.

.gitignore

@@ -31,4 +31,6 @@
 # Debug files
 *.dSYM/
 *.su
-*~
+*~
+
+pgsodium_encrypted_root.key

Dockerfile

@@ -11,6 +11,7 @@ COPY . .
 RUN make && make install
 RUN ldconfig
 RUN cd `pg_config --sharedir`/extension/
-RUN cp pgsodium_getkey.sample `pg_config --sharedir`/extension/pgsodium_getkey
+RUN cp getkey_scripts/pgsodium_getkey.sample `pg_config --sharedir`/extension/pgsodium_getkey
 RUN sed -i 's/exit//g' `pg_config --sharedir`/extension/pgsodium_getkey
 RUN chmod +x `pg_config --sharedir`/extension/pgsodium_getkey
+RUN cp `pg_config --sharedir`/extension/pgsodium_getkey /getkey

Dockerfile-debug

@@ -0,0 +1,28 @@
+ARG version
+FROM postgres:${version}
+ARG version
+
+# RUN apt-get update && apt-get install -y make git postgresql-server-dev-${version} curl build-essential gdb
+RUN apt-get update && apt-get install -y make git curl build-essential gdb libreadline-dev bison flex zlib1g-dev tmux zile zip gawk
+
+RUN git clone --branch REL_${version}_STABLE https://github.com/postgres/postgres.git --depth=1 && \
+    cd postgres && ./configure \
+    --prefix=/usr/ \
+    --enable-debug \
+    --enable-depend --enable-cassert --enable-profiling \
+    CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" \
+#    CFLAGS="-O3" \
+    && make -j 4 && make install
+
+RUN curl -s -L https://github.com/theory/pgtap/archive/v1.1.0.tar.gz | tar zxvf - && cd pgtap-1.1.0 && make && make install
+RUN curl -s -L https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz | tar zxvf - && cd libsodium-1.0.18 && ./configure && make check && make install
+RUN mkdir "/pgsodium"
+WORKDIR "/pgsodium"
+COPY . .
+RUN make && make install
+RUN ldconfig
+RUN curl -O https://raw.githubusercontent.com/tvondra/gdbpg/master/gdbpg.py
+RUN cd `pg_config --sharedir`/extension/
+RUN cp getkey_scripts/pgsodium_getkey.sample `pg_config --sharedir`/extension/pgsodium_getkey
+RUN sed -i 's/exit//g' `pg_config --sharedir`/extension/pgsodium_getkey
+RUN chmod +x `pg_config --sharedir`/extension/pgsodium_getkey

README.md

@@ -100,12 +100,25 @@ used without putting it in `shared_preload_libraries`, you will simply
 need to provide your own key management.  Skip ahead to the API usage
 section if you choose not to use server managed keys.
 
-See the file [`pgsodium_getkey.sample`](./pgsodium_getkey.sample) for
-an example script that returns a libsodium key.  The script must emit
-a hex encoded 32 byte (64 character) string on a single line.  DO NOT
-USE THIS FILE WITHOUT SUBSTITUTING YOUR OWN KEY.  Edit the file to add
-your own key and remove the `exit` line, remove the `.sample` suffix
-and make the file executable (on unixen `chmod +x pgsodium_getkey`).
+See the file
+[`getkey_scripts/pgsodium_getkey.sample`](./pgsodium_getkey.sample)
+for an example script that returns a libsodium key.  The script must
+emit a hex encoded 32 byte (64 character) string on a single line.  DO
+NOT USE THIS FILE WITHOUT SUBSTITUTING YOUR OWN KEY.  Edit the file to
+add your own key and remove the `exit` line, remove the `.sample`
+suffix and make the file executable (on unixen `chmod +x
+pgsodium_getkey`).
+
+pgsodium also comes with example scripts for:
+
+  - [Amazon Web Service's Key Management
+    Service](getkey_scripts/pgsodium_getkey_aws.sh).
+
+  - [Google Cloud's Cloud Key
+    Management](getkey_scripts/pgsodium_getkey_gcp.sh).
+
+  - [Zymbit Zymkey 4i Hardware Security
+    Module]((getkey_scripts/pgsodium_getkey_zmk.sh).
 
 Next place `pgsodium` in your `shared_preload_libraries`.  For docker
 containers, you can append this after the run:

example/Dockerfile

@@ -19,7 +19,7 @@ COPY . .
 RUN make && make install
 RUN ldconfig
 RUN cd `pg_config --sharedir`/extension/
-RUN cp pgsodium_getkey.sample `pg_config --sharedir`/extension/pgsodium_getkey
+RUN cp getkey_scripts/pgsodium_getkey.sample `pg_config --sharedir`/extension/pgsodium_getkey
 RUN sed -i 's/exit//g' `pg_config --sharedir`/extension/pgsodium_getkey
 RUN chmod +x `pg_config --sharedir`/extension/pgsodium_getkey
 RUN chown -R postgres:postgres /pgsodium

getkey_scripts/pgsodium_getkey_aws.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+HERE=`pwd`
+KEY_ID=${KEY_ID:-alias/pgsodium}
+ENCRYPTED_ROOT_KEY_FILE=${ENCRYPTED_ROOT_KEY_FILE:-$HERE/pgsodium_encrypted_root.key}
+
+if [[ -f "$ENCRYPTED_ROOT_KEY_FILE" ]]; then
+	aws kms decrypt --ciphertext-blob fileb://$ENCRYPTED_ROOT_KEY_FILE --query Plaintext --output text | base64 --decode | hex
+else
+	aws kms generate-data-key --number-of-bytes=32 --key-id=$KEY_ID --query CiphertextBlob --output text | base64 --decode > $ENCRYPTED_ROOT_KEY_FILE
+	aws kms decrypt --ciphertext-blob fileb://$ENCRYPTED_ROOT_KEY_FILE --query Plaintext --output text | base64 --decode | hex
+fi
+

getkey_scripts/pgsodium_getkey_gcp.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+HERE=`pwd`
+KEY=${KEY:-pgsodium}
+KEYRING=${KEYRING:-pgsodium}
+LOCATION=${LOCATION:-global}
+ROOT_KEY_FILE=${ROOT_KEY_FILE:-$HERE/pgsodium_encrypted_root.key}
+
+if [[ -f "$ROOT_KEY_FILE" ]]; then
+    gcloud kms decrypt          \
+           --key $KEY           \
+           --keyring $KEYRING   \
+           --location $LOCATION \
+           --plaintext-file -   \
+           --ciphertext-file $ROOT_KEY_FILE
+else
+    >&2 cat <<EOF 
+No root key file found at $ROOT_KEY_FILE for pgsodium to load.
+
+See
+https://cloud.google.com/kms/docs/creating-keys#kms-create-key-ring-cli
+to create a keyring and key.  Then encrypt a secret 32 byte payload
+with that key and save it to $ROOT_KEY_FILE.  For example, create a
+new keyring and key:
+
+    gcloud kms keyrings create pgsodium --location global
+    gcloud kms keys create pgsodium --keyring pgsodium --location global --purpose "encryption"
+
+Then encrypt a strong random key generated with pwgen into $ROOT_KEY_FILE:
+
+    pwgen 64 -s -1 -A -r ghijklmnopqrstuvwxyz | gcloud kms encrypt \\
+          --key pgsodium             \\
+          --keyring pgsodium         \\
+          --location global          \\
+          --plaintext-file - --ciphertext-file $ROOT_KEY_FILE
+
+Now restart postgres to initialize pgsodium with the new key.
+EOF
+    exit 1
+fi

getkey_scripts/pgsodium_getkey_zmk.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+HERE=`pwd`
+KEY=${KEY:-pgsodium}
+ROOT_KEY_FILE=${ROOT_KEY_FILE:-$HERE/pgsodium_encrypted_root.key}
+
+if [[ -f "$ROOT_KEY_FILE" ]]; then
+	python3 <<EOF
+from zymkey import Zymkey
+z = Zymkey()
+with open('$ROOT_KEY_FILE', 'rb') as f:
+	 z.unlock(f.read()).hex()
+EOF
+
+else
+    >&2 cat <<EEOF
+No root key file found at $ROOT_KEY_FILE for pgsodium to load.
+
+Using the zymkey API, encrypt (lock) a secret 32 byte payload and save
+it to $ROOT_KEY_FILE.  For example:
+
+pwgen 64 -s -1 -A -r ghijklmnopqrstuvwxyz | python3 <<EOF
+import zymkey, sys
+z = zymkey.Zymkey()
+with open('$ROOT_KEY_FILE', 'wb') as f:
+     f.write(z.lock(bytes.fromhex(sys.stdin.read())))
+EOF
+
+Now restart postgres to initialize pgsodium with the new key.
+EEOF
+    exit 1
+fi

psql.sh

@@ -6,12 +6,16 @@ DB_NAME="postgres"
 SU="postgres"
 EXEC="docker exec $DB_HOST"
 TAG="pgsodium/test-$version"
+CONFIG="-c shared_preload_libraries=pgsodium -c pgsodium.getkey_script=/getkey"
 
 echo building test image $DB_HOST
 docker build . -t $TAG --build-arg "version=$version"
 
 echo running test container
-docker run -v `pwd`/example:/pgsodium/example -e POSTGRES_HOST_AUTH_METHOD=trust -d --name "$DB_HOST" $TAG -c 'shared_preload_libraries=pgsodium' 
+docker run \
+	   -v `pwd`/example:/pgsodium/example \
+	   -e POSTGRES_HOST_AUTH_METHOD=trust \
+	   -d --name "$DB_HOST" $TAG $CONFIG
 
 echo waiting for database to accept connections
 until

src/pgsodium.c

@@ -3,67 +3,84 @@
 PG_MODULE_MAGIC;
 
 bytea* pgsodium_secret_key;
+static char *getkey_script = NULL;
+
 
 void _PG_init(void)
 {
-	FILE* fp;
-	char* secret_buf;
-	size_t secret_len = 0;
-	size_t char_read;
-	char* path;
-
-	char sharepath[MAXPGPATH];
+    FILE* fp;
+    char* secret_buf;
+    size_t secret_len = 0;
+    size_t char_read;
+    char* path;
+    char sharepath[MAXPGPATH];
 
-	if (sodium_init() == -1)
-	{
-		elog(ERROR, "_PG_init: sodium_init() failed cannot initialize pgsodium");
-		return;
-	}
+    if (sodium_init() == -1)
+    {
+        elog(ERROR, "_PG_init: sodium_init() failed cannot initialize pgsodium");
+        return;
+    }
 
-	if (process_shared_preload_libraries_in_progress)
-	{
-		get_share_path(my_exec_path, sharepath);
-		path = (char*) palloc(MAXPGPATH);
-		snprintf(
-			path,
-			MAXPGPATH,
-			"%s/extension/%s",
-			sharepath,
-			PG_GETKEY_EXEC);
+    if (process_shared_preload_libraries_in_progress)
+    {
+        path = (char*) palloc(MAXPGPATH);
+        get_share_path(my_exec_path, sharepath);
+        snprintf(
+                 path,
+                 MAXPGPATH,
+                 "%s/extension/%s",
+                 sharepath,
+                 PG_GETKEY_EXEC);
+        DefineCustomStringVariable(
+                 "pgsodium.getkey_script",
+                 "path to script that returns pgsodium root key",
+                 NULL,
+                 &getkey_script,
+                 path,
+                 PGC_POSTMASTER,
+                 0,
+                 NULL,
+                 NULL,
+                 NULL);
 
-		if (access(path, F_OK) == -1)
-			return;
+        if (access(getkey_script, F_OK) == -1)
+        {
+            fprintf(stderr,
+                    "Permission denied for %s\n",
+                    getkey_script);
+            proc_exit(1);
+        }
 
-		if ((fp = popen(path, "r")) == NULL)
-		{
-			fprintf(stderr,
-					"%s: could not launch shell command from\n",
-					path);
-			proc_exit(1);
-		}
+        if ((fp = popen(getkey_script, "r")) == NULL)
+        {
+            fprintf(stderr,
+                    "%s: could not launch shell command from\n",
+                    getkey_script);
+            proc_exit(1);
+        }
 
-		char_read = getline(&secret_buf, &secret_len, fp);
-		if (secret_buf[char_read-1] == '\n')
-			secret_buf[char_read-1] = '\0';
+        char_read = getline(&secret_buf, &secret_len, fp);
+        if (secret_buf[char_read-1] == '\n')
+            secret_buf[char_read-1] = '\0';
 
-		secret_len = strlen(secret_buf);
+        secret_len = strlen(secret_buf);
 
-		if (secret_len != 64)
-		{
-			fprintf(stderr, "invalid secret key\n");
-			proc_exit(1);
-		}
+        if (secret_len != 64)
+        {
+            fprintf(stderr, "invalid secret key\n");
+            proc_exit(1);
+        }
 
-		if (pclose(fp) != 0)
-		{
-			fprintf(stderr, "%s: could not close shell command\n",
-					PG_GETKEY_EXEC);
-			proc_exit(1);
-		}
-		pgsodium_secret_key = palloc(crypto_sign_SECRETKEYBYTES + VARHDRSZ);
-		hex_decode(secret_buf, secret_len, VARDATA(pgsodium_secret_key));
-		sodium_mlock(pgsodium_secret_key, crypto_sign_SECRETKEYBYTES + VARHDRSZ);
-		memset(secret_buf, 0, secret_len);
-		free(secret_buf);
-	}
+        if (pclose(fp) != 0)
+        {
+            fprintf(stderr, "%s: could not close shell command\n",
+                    PG_GETKEY_EXEC);
+            proc_exit(1);
+        }
+        pgsodium_secret_key = palloc(crypto_sign_SECRETKEYBYTES + VARHDRSZ);
+        hex_decode(secret_buf, secret_len, VARDATA(pgsodium_secret_key));
+        sodium_mlock(pgsodium_secret_key, crypto_sign_SECRETKEYBYTES + VARHDRSZ);
+        memset(secret_buf, 0, secret_len);
+        free(secret_buf);
+    }
 }

test.sh

@@ -6,7 +6,7 @@ versions=${1:-13 12 11 10}
 
 for version in $versions
 do
-	for config in '-c shared_preload_libraries=pgsodium' ''
+	for config in '-c shared_preload_libraries=pgsodium' '' '-c shared_preload_libraries=pgsodium -c pgsodium.getkey_script=/getkey'
 	do
 		DB_HOST="pgsodium-test-db-$version"
 		DB_NAME="postgres"
@@ -18,9 +18,10 @@ do
 		docker build . -t $TAG --build-arg "version=$version"
 
 		echo running test container
-		docker run -e POSTGRES_HOST_AUTH_METHOD=trust -d --name "$DB_HOST" $TAG $config
+		docker run --rm -e POSTGRES_HOST_AUTH_METHOD=trust -d --name "$DB_HOST" $TAG $config
 
 		echo waiting for database to accept connections
+		sleep 1;
 		until
 			$EXEC \
 				psql -o /dev/null -t -q -U "$SU" \