From e5f8bf65d19e05b7e97eb91148166e52ee572003 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Tue, 19 Apr 2022 16:08:05 +0000
Subject: [PATCH 01/15] Update Pg_exec to use explicitly allocated pgString.

---
 generic/pgtclCmds.c | 56 +++++++++++++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 15 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 3ebf995..0974a0f 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -254,6 +254,21 @@ int pgtclInitEncoding(Tcl_Interp *interp) {
 	return TCL_ERROR;
 }
 
+// Create a new "external" string from a "UTF" string
+char *makeExternalString(Tcl_Interp interp, char *utfString, int length)
+{
+	int newLength = 0;
+	if (length == -1) length = strlen(utfString);
+	char *externalString = ckalloc(length + 4 + 1); // 1 byte for null, 4 bytes to make Tcl_UtfToExternal happy
+
+	if (TCL_OK != Tcl_UtfToExternal(interp, utf8encoding, utfString, length, 0, NULL, externalString, length + 4 + 1, NULL, &newLength, NULL)) {
+		ckfree(externalString);
+		return NULL;
+	}
+	externalString[newLength] = '\0';
+	return externalString;
+}
+
 // The following two functions "waste" a DStrings storage by not freeing it until it's needed again
 // This is a little sloppy but massively simplifies the use since just about every place it's used
 // has to handle a possible early error return
@@ -802,7 +817,7 @@ Pg_exec(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 {
 	Pg_ConnectionId *connid;
 	PGconn	        *conn;
-	PGresult        *result;
+	PGresult        *result = NULL;
 	const char    *connString = NULL;
 	const char      *execString = NULL;
 	char            *newExecString = NULL;
@@ -902,18 +917,27 @@ Pg_exec(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 	    // After this point we must free paramValues and paramsBufferbefore exiting
         }
 
-	/* we could call PQexecParams when nParams is 0, but PQexecParams
-	 * will not accept more than one SQL statement per call, while
-	 * PQexec will.  by checking and using PQexec when no parameters
-	 * are included, we maintain compatibility for code that doesn't
-	 * use params and might have had multiple statements in a single
-	 * request */
-	if (nParams == 0) {
-	    result = PQexec(conn, externalString(execString));
-	} else {
-	    result = PQexecParams(conn, externalString(execString), nParams, NULL, paramValues, NULL, NULL, 0);
+	int validUTF = 0;
+	char *pgString = makeExternalString(execString);
+	if (pgString) {
+	    validUTF = 1;
+	    /* we could call PQexecParams when nParams is 0, but PQexecParams
+	     * will not accept more than one SQL statement per call, while
+	     * PQexec will.  by checking and using PQexec when no parameters
+	     * are included, we maintain compatibility for code that doesn't
+	     * use params and might have had multiple statements in a single
+	     * request */
+	    if (nParams == 0) {
+	        result = PQexec(conn, pgString);
+	    } else {
+	        result = PQexecParams(conn, pgString, nParams, NULL, paramValues, NULL, NULL, 0);
+	    }
 	}
 
+	if(pgString) {
+	    ckFree ((void *)pgString);
+	    sql = NULL;
+	}
 	if(paramValues) {
 	    ckfree ((void *)paramValues);
 	    paramValues = NULL;
@@ -954,11 +978,13 @@ Pg_exec(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 	}
 	else
 	{
-	    /* error occurred during the query */
-	    report_connection_error(interp, conn);
+	    if(validUTF) {
+		/* error occurred during the query */
+		report_connection_error(interp, conn);
 
-	    // Reconnect if the connection is bad.
-	    PgCheckConnectionState(connid);
+		// Reconnect if the connection is bad.
+		PgCheckConnectionState(connid);
+	    }
 
 	    return TCL_ERROR;
 	}

From 58bfa434b4e7b11ad334a489b0332f34d1bf75d7 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Tue, 19 Apr 2022 18:40:46 +0000
Subject: [PATCH 02/15] Initial conversion = pg_executePreopared, pg_execute,
 and pg_select

---
 generic/pgtclCmds.c | 79 ++++++++++++++++++++++++++++-----------------
 1 file changed, 49 insertions(+), 30 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 0974a0f..58fe38f 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -255,7 +255,7 @@ int pgtclInitEncoding(Tcl_Interp *interp) {
 }
 
 // Create a new "external" string from a "UTF" string
-char *makeExternalString(Tcl_Interp interp, char *utfString, int length)
+char *makeExternalString(Tcl_Interp *interp, const char *utfString, int length)
 {
 	int newLength = 0;
 	if (length == -1) length = strlen(utfString);
@@ -918,7 +918,7 @@ Pg_exec(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
         }
 
 	int validUTF = 0;
-	char *pgString = makeExternalString(execString);
+	char *pgString = makeExternalString(interp, execString, -1);
 	if (pgString) {
 	    validUTF = 1;
 	    /* we could call PQexecParams when nParams is 0, but PQexecParams
@@ -935,8 +935,8 @@ Pg_exec(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 	}
 
 	if(pgString) {
-	    ckFree ((void *)pgString);
-	    sql = NULL;
+	    ckfree ((void *)pgString);
+	    pgString = NULL;
 	}
 	if(paramValues) {
 	    ckfree ((void *)paramValues);
@@ -1030,7 +1030,7 @@ Pg_exec_prepared(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST
 {
 	Pg_ConnectionId *connid;
 	PGconn	   *conn;
-	PGresult   *result;
+	PGresult   *result = NULL;
 	const char	   *connString;
 	const char *statementNameString;
 	const char **paramValues = NULL;
@@ -1074,9 +1074,14 @@ Pg_exec_prepared(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST
 	    // After this point we must free paramValues and paramsBuffer before exiting
 	}
 
-	statementNameString = Tcl_GetString(objv[2]);
+	statementNameString = makeExternalString(interp, Tcl_GetString(objv[2]), -1);
+	int validUTF = statementNameString != NULL;
 
-	result = PQexecPrepared(conn, externalString(statementNameString), nParams, paramValues, NULL, NULL, 0);
+	if(statementNameString) {
+		result = PQexecPrepared(conn, statementNameString, nParams, paramValues, NULL, NULL, 0);
+		ckfree(statementNameString);
+		statementNameString = NULL;
+	}
 
 	if (paramValues != (const char **)NULL) {
 	    ckfree ((void *)paramValues);
@@ -1112,8 +1117,10 @@ Pg_exec_prepared(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST
 	}
 	else
 	{
-		/* error occurred during the query */
-		report_connection_error(interp, conn);
+		if(validUTF) {
+			/* error occurred during the query */
+			report_connection_error(interp, conn);
+		}
 
 		// Reconnect if the connection is bad.
 		PgCheckConnectionState(connid);
@@ -2068,11 +2075,17 @@ Pg_execute(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[]
                return TCL_ERROR;
         }
 
+	char *pgString = makeExternalString(interp, Tcl_GetString(objv[i++]), -1);
+	int validUTF = pgString != NULL;
 
-	/*
-	 * Execute the query
-	 */
-	result = PQexec(conn, externalString(Tcl_GetString(objv[i++])));
+	if(pgString) {
+		/*
+		 * Execute the query
+		 */
+		result = PQexec(conn, pgString);
+		ckfree(pgString);
+		pgString = NULL;
+	}
 	connid->sql_count++;
 
 	/*
@@ -2085,10 +2098,12 @@ Pg_execute(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[]
 	 */
 	if (result == NULL)
 	{
-		report_connection_error(interp, conn);
+		if(validUTF) {
+			report_connection_error(interp, conn);
 
-		// Look for a failed connection and re-open it.
-		PgCheckConnectionState(connid);
+			// Look for a failed connection and re-open it.
+			PgCheckConnectionState(connid);
+		}
 
 		return TCL_ERROR;
 	}
@@ -3117,7 +3132,7 @@ int
 Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 {
 	Pg_ConnectionId *connid;
-	PGconn	    *conn;
+	PGconn	    *conn = NULL;
 	PGresult    *result;
 	int          r,
 	             retval = TCL_ERROR;
@@ -3131,6 +3146,7 @@ Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 	int          index = 1;
 	int          nParams = 0;
 	char        *connString     = NULL;
+	char        *pgString       = NULL;
 	const char  *queryString    = NULL;
 	char        *varNameString  = NULL;
 	char        *paramArrayName = NULL;
@@ -3250,9 +3266,14 @@ Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 	    }
 	}
 
-	conn = PgGetConnectionId(interp, connString, &connid);
+	pgString = makeExternalString(interp, queryString, -1);
+
+	if(pgString)
+		conn = PgGetConnectionId(interp, connString, &connid);
+
 	if (conn == NULL) {
 	    cleanup_params_and_return_error: {
+		if(pgString) ckfree((void *)pgString);
 		if(paramValues) ckfree((void *)paramValues);
 		if(paramsBuffer) ckfree((void *)paramsBuffer);
 		if(newQueryString) ckfree((void *)newQueryString);
@@ -3260,21 +3281,17 @@ Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 	    }
 	}
 
-	if(nParams) {
-		// TODO convert parameters to external
-	}
-
 	connid->sql_count++;
+
 	if (rowByRow)
 	{
-		int status;
+		int status = 0;
 
 		// Make the call
 		if (nParams) {
-			status = PQsendQueryParams(conn, externalString(queryString), nParams,
-				NULL, paramValues, NULL, NULL, 0);
+			status = PQsendQueryParams(conn, pgString, nParams, NULL, paramValues, NULL, NULL, 0);
 		} else {
-			status = PQsendQuery(conn, externalString(queryString));
+			status = PQsendQuery(conn, pgString);
 		}
 
 		if(status == 0) {
@@ -3304,10 +3321,9 @@ Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 	} else {
 		// Make the call AND queue up the result.
 		if (nParams) {
-			result = PQexecParams(conn, externalString(queryString), nParams,
-				NULL, paramValues, NULL, NULL, 0);
+			result = PQexecParams(conn, pgString, nParams, NULL, paramValues, NULL, NULL, 0);
 		} else {
-			result = PQexec(conn, externalString(queryString));
+			result = PQexec(conn, pgString);
 		}
 
 		if (result == 0) {
@@ -3328,6 +3344,10 @@ Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 
 	// At this point we no longer need these. Zap them so we don't have to worry about them
 	// in the big loop.
+	if(pgString) {
+		ckfree((void *)pgString);
+		pgString = NULL;
+	}
 	if(paramValues) {
 		ckfree((void *)paramValues);
 		paramValues = NULL;
@@ -3336,7 +3356,6 @@ Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 		ckfree((void *)newQueryString);
 		newQueryString = NULL;
 	}
-
 	if(paramsBuffer) {
 		ckfree((void *)paramsBuffer);
 		paramsBuffer = NULL;

From 264accd55cd03be04b42e0676704a017f125c3b3 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Wed, 20 Apr 2022 15:39:06 +0000
Subject: [PATCH 03/15] Converting more functions.

---
 generic/pgtclCmds.c | 112 ++++++++++++++++++++++++--------------------
 1 file changed, 62 insertions(+), 50 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 58fe38f..8e4c49a 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -1982,12 +1982,12 @@ Pg_execute(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[]
 {
 	Pg_ConnectionId *connid;
 	PGconn	   *conn;
-	PGresult   *result;
-	int			i;
-	int			tupno;
-	int			ntup;
-	int			loop_rc;
-	const char	   *array_varname = NULL;
+	PGresult   *result = NULL;
+	int        i;
+	int        tupno;
+	int        ntup;
+	int        loop_rc;
+	const char *array_varname = NULL;
 	char	   *arg;
 
 	Tcl_Obj    *oid_varnameObj = NULL;
@@ -3795,7 +3795,7 @@ Pg_sendquery(ClientData cData, Tcl_Interp *interp, int objc,
 {
 	Pg_ConnectionId *connid;
 	PGconn	        *conn;
-        int              status;
+        int              status = 0;
 	const char    *connString = NULL;
 	const char      *execString = NULL;
 	char            *newExecString = NULL;
@@ -3895,10 +3895,20 @@ Pg_sendquery(ClientData cData, Tcl_Interp *interp, int objc,
 	    }
         }
 
-	if (nParams == 0) {
-	    status = PQsendQuery(conn, externalString(execString));
-	} else {
-	    status = PQsendQueryParams(conn, externalString(execString), nParams, NULL, paramValues, NULL, NULL, 1);
+	char *pgString = makeExternalString(interp, execString, -1);
+	int validUTF = pgString != NULL;
+
+	if(pgString) {
+	    if (nParams == 0) {
+		status = PQsendQuery(conn, pgString);
+	    } else {
+		status = PQsendQueryParams(conn, pgString, nParams, NULL, paramValues, NULL, NULL, 1);
+	    }
+	}
+
+	if(pgString) {
+	    ckfree(pgString);
+	    pgString = NULL;
 	}
 	if(newExecString) {
 	    ckfree(newExecString);
@@ -3921,11 +3931,13 @@ Pg_sendquery(ClientData cData, Tcl_Interp *interp, int objc,
 	    return TCL_OK;
 	else
 	{
-	    /* error occurred during the query */
-	    report_connection_error(interp, conn);
+	    if(validUTF) {
+		/* error occurred during the query */
+		report_connection_error(interp, conn);
 
-	    // Reconnect if the connection is bad.
-	    PgCheckConnectionState(connid);
+		// Reconnect if the connection is bad.
+		PgCheckConnectionState(connid);
+	    }
 
 	    return TCL_ERROR;
 	}
@@ -5435,32 +5447,32 @@ Pg_sql(ClientData cData, Tcl_Interp *interp, int objc,
     /*
      *  Handle param options
      */
-     if (params) {
-         Tcl_ListObjGetElements(interp, objv[binparams], &countbin, &elembinPtrs);
+    if (params) {
+        Tcl_ListObjGetElements(interp, objv[binparams], &countbin, &elembinPtrs);
 
-         if (countbin != 0 && countbin != count) {
+        if (countbin != 0 && countbin != count) {
             Tcl_SetResult(interp, "-params and -binparams need the same number of elements", TCL_STATIC); 
             return TCL_ERROR;
-         }
+        }
 
-	 int param;
+	int param;
 
-	 paramValues = (const char **)ckalloc (count * sizeof (char *));
-	 binValues = (int *)ckalloc (countbin * sizeof (char *));
+	paramValues = (const char **)ckalloc (count * sizeof (char *));
+	binValues = (int *)ckalloc (countbin * sizeof (char *));
 
-	 for (param = 0; param < count; param++) {
+	for (param = 0; param < count; param++) {
 		// TODO convert to external
-	     paramValues[param] = Tcl_GetString (elemPtrs[param]);
-	     if (strcmp(paramValues[param], "NULL") == 0)
-             {
-                 paramValues[param] = NULL;
-             }
-	 }
-
-	 for (param = 0; param < countbin; param++) {
-	     Tcl_GetBooleanFromObj (interp, elembinPtrs[param], &binValues[param]);
-	 }
-     }
+	    paramValues[param] = Tcl_GetString (elemPtrs[param]);
+	    if (strcmp(paramValues[param], "NULL") == 0)
+            {
+                paramValues[param] = NULL;
+            }
+	}
+
+	for (param = 0; param < countbin; param++) {
+	    Tcl_GetBooleanFromObj (interp, elembinPtrs[param], &binValues[param]);
+	}
+    }
 
     connString = Tcl_GetString(objv[1]);
     conn = PgGetConnectionId(interp, connString, &connid);
@@ -5483,37 +5495,37 @@ Pg_sql(ClientData cData, Tcl_Interp *interp, int objc,
         {
             Tcl_SetResult(interp, "Attempt to wait for result while already waiting", TCL_STATIC);
             return TCL_ERROR;
-       }
+        }
 
-       /* Start the notify event source if it isn't already running */
-       PgStartNotifyEventSource(connid);
+        /* Start the notify event source if it isn't already running */
+        PgStartNotifyEventSource(connid);
 
-       connid->callbackPtr= objv[callback];
-       connid->callbackInterp= interp;
+        connid->callbackPtr= objv[callback];
+        connid->callbackInterp= interp;
 
-       Tcl_IncrRefCount(objv[callback]);
-       Tcl_Preserve((ClientData) interp);
+        Tcl_IncrRefCount(objv[callback]);
+        Tcl_Preserve((ClientData) interp);
 
-       /* 
-        *  invoke function based on type 
-        *  of query 
-        */
+        /* 
+         *  invoke function based on type 
+         *  of query 
+         */
         if (prepared) {
-	    iResult = PQsendQueryPrepared(conn, externalString(execString), count, paramValues, paramLengths, binValues, binresults);
+            iResult = PQsendQueryPrepared(conn, externalString(execString), count, paramValues, paramLengths, binValues, binresults);
         } else if (params) {
             iResult = PQsendQueryParams(conn, externalString(execString), count, NULL, paramValues, paramLengths, binValues, binresults);
 
         } else {
     
-            iResult = PQsendQuery(conn, externalString(execString));
+             iResult = PQsendQuery(conn, externalString(execString));
 /*
-            ckfree ((void *)paramValues);
+             ckfree ((void *)paramValues);
 */
-        }
+         }
     } else {
 
         if (prepared) {
-	    result = PQexecPrepared(conn, externalString(execString), count, paramValues, paramLengths, binValues, binresults);
+            result = PQexecPrepared(conn, externalString(execString), count, paramValues, paramLengths, binValues, binresults);
         } else if (params) {
             result = PQexecParams(conn, externalString(execString), count, NULL, paramValues, paramLengths, binValues, binresults);
         } else {

From 7f132b2cb844e6f9613071d3c077bc0165d531ea Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Wed, 20 Apr 2022 18:44:34 +0000
Subject: [PATCH 04/15] Clarify comment.

---
 generic/pgtclCmds.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 8e4c49a..7f01973 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -706,7 +706,8 @@ Pg_disconnect(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST obj
 
 /* helper for build_param_array and other related functions.
 ** convert nParams strings in paramValues, lengths in paramLengths,
-** return allocated buffer containing new strings in bufferPtr
+** allocates and returns buffer containing new strings in bufferPtr
+** for later disposal.
 */
 int array_to_utf8(Tcl_Interp *interp, const char **paramValues, int *paramLengths, int nParams, const char **bufferPtr)
 {

From 3f4e5ad0525a9444ac65c90bc81673f977bcc4c6 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Thu, 21 Apr 2022 13:34:44 +0000
Subject: [PATCH 05/15] Convert Pg_sql

---
 generic/pgtclCmds.c | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 7f01973..01acf36 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -5486,18 +5486,23 @@ Pg_sql(ClientData cData, Tcl_Interp *interp, int objc,
         return TCL_ERROR;
     }
 
-    execString = Tcl_GetString(objv[2]);
-
-    /*
-     * Handle the callback first, before executing statments
-     */
     if (callback) {
         if (connid->callbackPtr || connid->callbackInterp)
         {
             Tcl_SetResult(interp, "Attempt to wait for result while already waiting", TCL_STATIC);
             return TCL_ERROR;
         }
+    }
+
+    execString = Tcl_GetString(objv[2]);
+    if(!execString) {
+	return TCL_ERROR;
+    }
 
+    /*
+     * Handle the callback first, before executing statments
+     */
+    if (callback) {
         /* Start the notify event source if it isn't already running */
         PgStartNotifyEventSource(connid);
 
@@ -5512,13 +5517,13 @@ Pg_sql(ClientData cData, Tcl_Interp *interp, int objc,
          *  of query 
          */
         if (prepared) {
-            iResult = PQsendQueryPrepared(conn, externalString(execString), count, paramValues, paramLengths, binValues, binresults);
+            iResult = PQsendQueryPrepared(conn, execString, count, paramValues, paramLengths, binValues, binresults);
         } else if (params) {
-            iResult = PQsendQueryParams(conn, externalString(execString), count, NULL, paramValues, paramLengths, binValues, binresults);
+            iResult = PQsendQueryParams(conn, execString, count, NULL, paramValues, paramLengths, binValues, binresults);
 
         } else {
     
-             iResult = PQsendQuery(conn, externalString(execString));
+             iResult = PQsendQuery(conn, execString);
 /*
              ckfree ((void *)paramValues);
 */
@@ -5526,15 +5531,18 @@ Pg_sql(ClientData cData, Tcl_Interp *interp, int objc,
     } else {
 
         if (prepared) {
-            result = PQexecPrepared(conn, externalString(execString), count, paramValues, paramLengths, binValues, binresults);
+            result = PQexecPrepared(conn, execString, count, paramValues, paramLengths, binValues, binresults);
         } else if (params) {
-            result = PQexecParams(conn, externalString(execString), count, NULL, paramValues, paramLengths, binValues, binresults);
+            result = PQexecParams(conn, execString, count, NULL, paramValues, paramLengths, binValues, binresults);
         } else {
-            result = PQexec(conn, externalString(execString));
+            result = PQexec(conn, execString);
             ckfree ((void *)paramValues);
         }
     } /* end if callback */
 
+    ckfree(execString);
+    execString = NULL;
+
     PgNotifyTransferEvents(connid);
 
     // Reconnect if the connection is bad.
@@ -5567,7 +5575,6 @@ Pg_sql(ClientData cData, Tcl_Interp *interp, int objc,
 	return TCL_ERROR;
     }
 
-    
     return TCL_OK;
 }
 

From bacb15d7fff1d1d42466dc21be2aa23473ab49e2 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Thu, 21 Apr 2022 14:45:02 +0000
Subject: [PATCH 06/15] Remove old externalString function and add error
 message for makeExternalString

---
 generic/pgtclCmds.c | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 01acf36..3c3dfda 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -261,10 +261,18 @@ char *makeExternalString(Tcl_Interp *interp, const char *utfString, int length)
 	if (length == -1) length = strlen(utfString);
 	char *externalString = ckalloc(length + 4 + 1); // 1 byte for null, 4 bytes to make Tcl_UtfToExternal happy
 
-	if (TCL_OK != Tcl_UtfToExternal(interp, utf8encoding, utfString, length, 0, NULL, externalString, length + 4 + 1, NULL, &newLength, NULL)) {
+	int code = Tcl_UtfToExternal(interp, utf8encoding, utfString, length, 0, NULL, externalString, length + 4 + 1, NULL, &newLength, NULL);
+
+	if (code != TCL_OK) {
+		static char errmsg[128];
 		ckfree(externalString);
+
+		sprintf(errmsg, "Error %d attempting to convert '%.40s...' to external utf8", code, utfString);
+		Tcl_SetResult(interp, errmsg, TCL_VOLATILE);
+
 		return NULL;
 	}
+
 	externalString[newLength] = '\0';
 	return externalString;
 }
@@ -274,18 +282,6 @@ char *makeExternalString(Tcl_Interp *interp, const char *utfString, int length)
 // has to handle a possible early error return
 // NOTE: only one of these can ever be in flight at any time.
 
-/*
- * Convert one utf string at a time to an external string, hiding the DString management.
- */
-char *externalString(const char *utfString)
-{
-	static Tcl_DString tmpds;
-	static int allocated = 0;
-	if(allocated) Tcl_DStringFree(&tmpds);
-	allocated = 1;
-	return Tcl_UtfToExternalDString(utf8encoding, utfString, -1, &tmpds);
-}
-
 /*
  * Convert one external string at a time to a utf string, hiding the DString management.
  */

From eea6764ac03e5205ebd8745db3e0d840ce294dc6 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Tue, 26 Apr 2022 13:55:51 +0000
Subject: [PATCH 07/15] First pass at makeUTFString.

---
 generic/pgtclCmds.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 3c3dfda..5db1af9 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -277,6 +277,34 @@ char *makeExternalString(Tcl_Interp *interp, const char *utfString, int length)
 	return externalString;
 }
 
+// Create a new "UTF" string from an "external" string.
+char *makeUTFString(Tcl_Interp *interp, const char *externalString, int length)
+{
+	int newLength = 0;
+	if (length == -1) length = strlen(externalString);
+
+	// Since this may convert a code point to a surrogate pair, worst case is the string
+	// length doubling, plus 4 bytes to make Tcl_ExternalToUtf happy, plus a terminating null;
+	int bufferSize = length * 2 + 4 + 1;
+
+	char *UTFString = ckalloc(bufferSize);
+
+	int code = Tcl_ExternalToUtf(interp, utf8encoding, externalString, length, 0, NULL, UTFString, bufferSize, NULL, &newLength, NULL);
+
+	if(code != TCL_OK) {
+		static char errmsg[128];
+		ckfree(UTFString);
+
+		sprintf(errmsg, "Error %d attempting to convert '%.40s...' to internal UTF", code, externalString);
+		Tcl_SetResult(interp, errmsg, TCL_VOLATILE);
+
+		return NULL;
+	}
+
+	UTFString[newLength] = '\0';
+	return UTFString;
+}
+
 // The following two functions "waste" a DStrings storage by not freeing it until it's needed again
 // This is a little sloppy but massively simplifies the use since just about every place it's used
 // has to handle a possible early error return

From d3a7ec9ebc51c2ea6a6062369bbf980076489079 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Tue, 26 Apr 2022 15:35:47 +0000
Subject: [PATCH 08/15] Create UTF_SetVar2 and UTF_ObjSetVar2 and start
 converting pq_Select etc to use them.

---
 generic/pgtclCmds.c | 86 +++++++++++++++++++++++++++++----------------
 1 file changed, 56 insertions(+), 30 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index 5db1af9..b18fe83 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -305,6 +305,36 @@ char *makeUTFString(Tcl_Interp *interp, const char *externalString, int length)
 	return UTFString;
 }
 
+// helper function, converts an external string to Tcl UTF and passes it to Tcl_SetVar
+const char *UTF_SetVar2(Tcl_Interp *interp, const char *name1, const char *name2, const char *newValue, int flags)
+{
+	char *UTFnewValue = makeUTFString(interp, newValue, -1);
+
+	if(!UTFnewValue) return NULL;
+
+	const char *code = Tcl_SetVar2(interp, name1, name2, newValue, flags);
+
+	ckfree(UTFnewValue);
+
+	return code;
+}
+
+
+// helper function, converts an external string to Tcl UTF, creates an Object, and passes it to Tcl_SetVarObj
+// Not quite compatible with Tcl_ObjSetVar2
+Tcl_Obj *UTF_ObjSetVar2(Tcl_Interp *interp, Tcl_Obj *part1ptr, Tcl_Obj *part2ptr, const char *newValue, int flags)
+{
+	char *UTFnewValue = makeUTFString(interp, newValue, -1);
+
+	if(!UTFnewValue) return NULL;
+
+	Tcl_Obj *resultPtr = Tcl_ObjSetVar2(interp, part1Ptr, part2Ptr, Tcl_NewStringObj(UTFnewValue, -1), flags);
+
+	ckfree(UTFnewValue);
+
+	return resultPtr;
+}
+
 // The following two functions "waste" a DStrings storage by not freeing it until it's needed again
 // This is a little sloppy but massively simplifies the use since just about every place it's used
 // has to handle a possible early error return
@@ -1187,9 +1217,7 @@ Pg_result_foreach(Tcl_Interp *interp, PGresult *result, Tcl_Obj *arrayNameObj, T
 			continue;
 		    }
 
-		    char *string = PQgetvalue (result, tupno, column);
-
-		    if (Tcl_SetVar2(interp, arrayName, columnName, utfString(string), (TCL_LEAVE_ERR_MSG)) == NULL) 
+		    if (UTF_SetVar2(interp, arrayName, columnName, PQgetvalue (result, tupno, column), (TCL_LEAVE_ERR_MSG)) == NULL) 
 		    {
 			return TCL_ERROR;
 		    }
@@ -1544,11 +1572,7 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 						Tcl_AppendToObj(fieldNameObj, ",", 1);
 						Tcl_AppendToObj(fieldNameObj, PQfname(result, i), -1);
 
-
-						if (Tcl_ObjSetVar2(interp, arrVarObj, fieldNameObj,
-										   Tcl_NewStringObj(
-											 utfString(PGgetvalue(result, resultid->nullValueString, tupno, i)),
-											 -1), TCL_LEAVE_ERR_MSG) == NULL) {
+						if (UTF_ObjSetVar2(interp, arrVarObj, fieldNameObj, PGgetvalue(result, resultid->nullValueString, tupno, i), TCL_LEAVE_ERR_MSG) == NULL) {
 							Tcl_DecrRefCount (fieldNameObj);
 							return TCL_ERROR;
 						}
@@ -1583,33 +1607,32 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 				 */
 				for (tupno = 0; tupno < PQntuples(result); tupno++)
 				{
-					const char *field0 = utfString(PGgetvalue(result, resultid->nullValueString, tupno, 0));
-					char *dupfield0 = ckalloc(strlen(field0)+1);
-
-					strcpy(dupfield0, field0);
+					const char *field0 = makeUTFString(interp, PGgetvalue(result, resultid->nullValueString, tupno, 0), -1);
 
 					for (i = 1; i < PQnfields(result); i++)
 					{
 						Tcl_Obj    *fieldNameObj;
 
 						fieldNameObj = Tcl_NewObj ();
-						Tcl_SetStringObj(fieldNameObj, dupfield0, -1);
+						Tcl_SetStringObj(fieldNameObj, field0, -1);
 						Tcl_AppendToObj(fieldNameObj, ",", 1);
 						Tcl_AppendToObj(fieldNameObj, PQfname(result, i), -1);
 
 						if (appendstrObj != NULL)
 							Tcl_AppendObjToObj(fieldNameObj, appendstrObj);
 
-						char *val = utfString(PGgetvalue(result, resultid->nullValueString, tupno, i));
-						if (Tcl_ObjSetVar2(interp, arrVarObj, fieldNameObj,
-						      Tcl_NewStringObj( val, -1), TCL_LEAVE_ERR_MSG) == NULL)
+						if (UTF_ObjSetVar2(
+							interp, arrVarObj, fieldNameObj,
+							PGgetvalue(result, resultid->nullValueString, tupno, i), -1),
+							TCL_LEAVE_ERR_MSG
+						) == NULL)
 						{
 							Tcl_DecrRefCount(fieldNameObj);
-							ckfree(dupfield0);
+							ckfree(field0);
 							return TCL_ERROR;
 						}
 					}
-					ckfree(dupfield0);
+					ckfree(field0);
 				}
 				return TCL_OK;
 			}
@@ -1646,13 +1669,20 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 				/* build up a return list, Tcl-object-style */
 				for (i = 0; i < PQnfields(result); i++)
 				{
-					char	   *value;
+					char *value = makeUTFString(
+						interp,
+						PGgetvalue(result, resultid->nullValueString, tupno, i),
+						-1);
+
+					if(!value) return TCL_ERROR;
 
-					value = utfString(PGgetvalue(result, resultid->nullValueString, tupno, i));
+					Tcl_Obj *valueObj = Tcl_NewStringObj(value, -1)
 
-					if (Tcl_ListObjAppendElement(interp, resultObj, 
-							   Tcl_NewStringObj(value, -1)) == TCL_ERROR)
+					if (Tcl_ListObjAppendElement(interp, resultObj, valueObj) == TCL_ERROR) {
+						Tcl_DecrRefCount(valueObj);
+						ckfree(value);
 						return TCL_ERROR;
+					}
 				}
 				Tcl_SetObjResult(interp, resultObj);
 				return TCL_OK;
@@ -1689,11 +1719,9 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 					 */
 					for (i = 0; i < PQnfields(result); i++)
 					{
-						if (Tcl_SetVar2(interp, arrayName, PQfname(result, i),
-							utfString(
-							    PGgetvalue(result, resultid->nullValueString, 
-								tupno, i)
-							), TCL_LEAVE_ERR_MSG) == NULL)
+						if (UTF_SetVar2(interp, arrayName, PQfname(result, i),
+							PGgetvalue(result, resultid->nullValueString, tupno, i),
+							TCL_LEAVE_ERR_MSG) == NULL)
 						return TCL_ERROR;
 					}
 				} else
@@ -1714,9 +1742,7 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 							}
 						}
 
-						if (Tcl_SetVar2(interp, arrayName, PQfname(result, i),
-									 utfString(string),
-										TCL_LEAVE_ERR_MSG) == NULL)
+						if (UTF_SetVar2(interp, arrayName, PQfname(result, i), string, TCL_LEAVE_ERR_MSG) == NULL)
 							return TCL_ERROR;
 					}
 				}

From adefee23d9341459d7ce0e21fba3bd796cebc8cc Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Wed, 27 Apr 2022 14:50:48 +0000
Subject: [PATCH 09/15] Convert pg_result.

---
 generic/pgtclCmds.c | 45 ++++++++++++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index b18fe83..db57e32 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -328,7 +328,7 @@ Tcl_Obj *UTF_ObjSetVar2(Tcl_Interp *interp, Tcl_Obj *part1ptr, Tcl_Obj *part2ptr
 
 	if(!UTFnewValue) return NULL;
 
-	Tcl_Obj *resultPtr = Tcl_ObjSetVar2(interp, part1Ptr, part2Ptr, Tcl_NewStringObj(UTFnewValue, -1), flags);
+	Tcl_Obj *resultPtr = Tcl_ObjSetVar2(interp, part1ptr, part2ptr, Tcl_NewStringObj(UTFnewValue, -1), flags);
 
 	ckfree(UTFnewValue);
 
@@ -1623,9 +1623,9 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 
 						if (UTF_ObjSetVar2(
 							interp, arrVarObj, fieldNameObj,
-							PGgetvalue(result, resultid->nullValueString, tupno, i), -1),
+							PGgetvalue(result, resultid->nullValueString, tupno, i),
 							TCL_LEAVE_ERR_MSG
-						) == NULL)
+						    ) == NULL)
 						{
 							Tcl_DecrRefCount(fieldNameObj);
 							ckfree(field0);
@@ -1676,7 +1676,7 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 
 					if(!value) return TCL_ERROR;
 
-					Tcl_Obj *valueObj = Tcl_NewStringObj(value, -1)
+					Tcl_Obj *valueObj = Tcl_NewStringObj(value, -1);
 
 					if (Tcl_ListObjAppendElement(interp, resultObj, valueObj) == TCL_ERROR) {
 						Tcl_DecrRefCount(valueObj);
@@ -1827,10 +1827,16 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 				*/
 				for (i = 0; i < PQnfields(result); i++)
 				{
-				    fieldObj = Tcl_NewObj();
+					char *fieldString = makeUTFString(interp, PGgetvalue(result, resultid->nullValueString, tupno, i), -1);
+					if(!fieldString) {
+						Tcl_DecrRefCount(listObj);
+						return TCL_ERROR;
+					}
 
-				    Tcl_SetStringObj(fieldObj, utfString(PGgetvalue(result, resultid->nullValueString, tupno, i)), -1);
-				    if (Tcl_ListObjAppendElement(interp, listObj, fieldObj) != TCL_OK)
+					fieldObj = Tcl_NewStringObj(fieldString, -1);
+					ckfree(fieldString);
+
+					if (Tcl_ListObjAppendElement(interp, listObj, fieldObj) != TCL_OK)
 					{
 						Tcl_DecrRefCount(listObj);
 						Tcl_DecrRefCount(fieldObj);
@@ -1868,10 +1874,14 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 				*/
 				for (i = 0; i < PQnfields(result); i++)
 				{
-	
-					fieldObj = Tcl_NewObj();
+					char *fieldString = makeUTFString(interp, PGgetvalue(result, resultid->nullValueString, tupno, i), -1);
+					if(!fieldString) {
+						Tcl_DecrRefCount(listObj);
+						return TCL_ERROR;
+					}
 
-					Tcl_SetStringObj(fieldObj, utfString(PGgetvalue(result, resultid->nullValueString, tupno, i)), -1);
+					fieldObj = Tcl_NewStringObj(fieldString, -1);
+					ckfree(fieldString);
 	
 					if (Tcl_ListObjAppendElement(interp, subListObj, fieldObj) != TCL_OK)
 					{
@@ -1917,17 +1927,22 @@ Pg_result(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 				*/
 				for (i = 0; i < PQnfields(result); i++)
 				{
-	
-					fieldObj = Tcl_NewObj();
-					fieldNameObj = Tcl_NewObj();
+					char *fieldString = makeUTFString(interp, PGgetvalue(result, resultid->nullValueString, tupno, i), -1);
+					if(!fieldString) {
+						Tcl_DecrRefCount(listObj);
+						return TCL_ERROR;
+					}
 
-					Tcl_SetStringObj(fieldNameObj, PQfname(result, i), -1);
-					Tcl_SetStringObj(fieldObj, utfString(PGgetvalue(result, resultid->nullValueString, tupno, i)), -1);
+					fieldObj = Tcl_NewStringObj(fieldString, -1);
+					ckfree(fieldString);
+	
+					fieldNameObj = Tcl_NewStringObj(PQfname(result, i), -1);
 	
 					if (Tcl_DictObjPut(interp, subListObj, fieldNameObj, fieldObj) != TCL_OK)
 					{
 						Tcl_DecrRefCount(listObj);
 						Tcl_DecrRefCount(fieldObj);
+						Tcl_DecrRefCount(fieldNameObj);
 						return TCL_ERROR;
 					}
 	

From 5a74fb576723555b99cc9746c5c47d08383d6950 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Wed, 27 Apr 2022 18:31:55 +0000
Subject: [PATCH 10/15] Final conversion of utfString() to makeUTFString. Phew.

---
 generic/pgtclCmds.c | 45 ++++++++++++++++++++-------------------------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/generic/pgtclCmds.c b/generic/pgtclCmds.c
index db57e32..779648a 100644
--- a/generic/pgtclCmds.c
+++ b/generic/pgtclCmds.c
@@ -305,7 +305,7 @@ char *makeUTFString(Tcl_Interp *interp, const char *externalString, int length)
 	return UTFString;
 }
 
-// helper function, converts an external string to Tcl UTF and passes it to Tcl_SetVar
+// helper function, converts an external string to Tcl UTF and passes it to Tcl_SetVar2
 const char *UTF_SetVar2(Tcl_Interp *interp, const char *name1, const char *name2, const char *newValue, int flags)
 {
 	char *UTFnewValue = makeUTFString(interp, newValue, -1);
@@ -320,7 +320,7 @@ const char *UTF_SetVar2(Tcl_Interp *interp, const char *name1, const char *name2
 }
 
 
-// helper function, converts an external string to Tcl UTF, creates an Object, and passes it to Tcl_SetVarObj
+// helper function, converts an external string to Tcl UTF, creates an Object, and passes it to Tcl_ObjSetVar2
 // Not quite compatible with Tcl_ObjSetVar2
 Tcl_Obj *UTF_ObjSetVar2(Tcl_Interp *interp, Tcl_Obj *part1ptr, Tcl_Obj *part2ptr, const char *newValue, int flags)
 {
@@ -335,25 +335,6 @@ Tcl_Obj *UTF_ObjSetVar2(Tcl_Interp *interp, Tcl_Obj *part1ptr, Tcl_Obj *part2ptr
 	return resultPtr;
 }
 
-// The following two functions "waste" a DStrings storage by not freeing it until it's needed again
-// This is a little sloppy but massively simplifies the use since just about every place it's used
-// has to handle a possible early error return
-// NOTE: only one of these can ever be in flight at any time.
-
-/*
- * Convert one external string at a time to a utf string, hiding the DString management.
- */
-char *utfString(const char *externalString)
-{
-	static Tcl_DString tmpds;
-	static int allocated = 0;
-	if(allocated) Tcl_DStringFree(&tmpds);
-	allocated = 1;
-	return Tcl_ExternalToUtfDString(utf8encoding, externalString, -1, &tmpds);
-}
-
-// TODO something to simplify an array worth of external or utf strings in flight at a time
-
 /*
  * PGgetvalue()
  *
@@ -2322,19 +2303,27 @@ execute_put_values(Tcl_Interp *interp, const char *array_varname,
 	for (i = 0; i < n; i++)
 	{
 		fname = PQfname(result, i);
-		value = utfString(PGgetvalue(result, nullValueString, tupno, i));
+		value = makeUTFString(interp, PGgetvalue(result, nullValueString, tupno, i), -1);
+		if(!value) {
+			return TCL_ERROR;
+		}
 
 		if (array_varname != NULL)
 		{
 			if (Tcl_SetVar2(interp, array_varname, fname, value,
-							TCL_LEAVE_ERR_MSG) == NULL)
+							TCL_LEAVE_ERR_MSG) == NULL) {
+				ckfree(value);
 				return TCL_ERROR;
+			}
 		}
 		else
 		{
-			if (Tcl_SetVar(interp, fname, value, TCL_LEAVE_ERR_MSG) == NULL)
+			if (Tcl_SetVar(interp, fname, value, TCL_LEAVE_ERR_MSG) == NULL) {
+				ckfree(value);
 				return TCL_ERROR;
+			}
 		}
+		ckfree(value);
 	}
 	return TCL_OK;
 }
@@ -3543,7 +3532,13 @@ Pg_select(ClientData cData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[])
 				}
 
 				if (valueObj == NULL) {
-					valueObj = Tcl_NewStringObj(utfString(string), -1);
+					char *utf = makeUTFString(interp, string, -1);
+					if(!utf) {
+						retval = TCL_ERROR;
+						goto done;
+					}
+					valueObj = Tcl_NewStringObj(utf, -1);
+					ckfree(utf);
 				}
 
 				if (Tcl_ObjSetVar2(interp, varNameObj, columnNameObjs[column],

From 1e0d3ad01819714303dc3168244258dec3111a95 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Wed, 27 Apr 2022 19:03:43 +0000
Subject: [PATCH 11/15] Update version to anticipated final version #

---
 configure.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.in b/configure.in
index dfaed0c..a256a33 100755
--- a/configure.in
+++ b/configure.in
@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so you can encode the package version directly into the source files.
 #-----------------------------------------------------------------------
 
-AC_INIT([pgtcl], [2.7.7])
+AC_INIT([pgtcl], [2.8.1])
 
 #-----
 # Version with patch stripped

From 935cd2686424dde8d6413730f80bb1c438365f75 Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Fri, 29 Apr 2022 14:28:39 +0000
Subject: [PATCH 12/15] Fix test where it was setting an array element rather
 than a variable.

---
 tests/pgtcl.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/pgtcl.test b/tests/pgtcl.test
index 7a18260..a5e0663 100644
--- a/tests/pgtcl.test
+++ b/tests/pgtcl.test
@@ -1344,7 +1344,7 @@ test pgtcl-11.3 {using pg_exec with 4-byte unicode positional args} -body {
 
     set conn [pg::connect -connlist [array get ::conninfo]]
 
-    set paramarray(glyph) "𝔄"
+    set glyph "𝔄"
 
     set res [$conn exec {SELECT relname
                            FROM Pg_class

From 06f5cea6a4415d81547d2dc58e7f2aa5f0a862dd Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Mon, 2 May 2022 16:36:23 +0000
Subject: [PATCH 13/15] Add bulk tests.

---
 tests/pgtcl-bulk.test | 407 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 407 insertions(+)
 create mode 100644 tests/pgtcl-bulk.test

diff --git a/tests/pgtcl-bulk.test b/tests/pgtcl-bulk.test
new file mode 100644
index 0000000..553c777
--- /dev/null
+++ b/tests/pgtcl-bulk.test
@@ -0,0 +1,407 @@
+if {[lsearch [namespace children] ::tcltest] == -1} {
+    package require tcltest 2
+    namespace import -force ::tcltest::*
+}
+
+#tcltest::verbose 100
+#tcltest::debug 100
+
+set flist [glob -nocomplain -dir .. libpgtcl*[info sharedlibextension]]
+set flist [concat $flist [glob -nocomplain libpgtcl*[info sharedlibextension]]]
+
+if {[llength $flist] == 0} {
+    puts "\nCan not find a shared lib file\n"
+    exit
+}
+
+
+puts [list loading [lindex $flist end] ...]
+load [lindex $flist end]
+
+if [file exists $env(HOME)/.conninfo.tcl] {
+    puts [list sourcing $env(HOME)/.conninfo.tcl ...]
+    source $env(HOME)/.conninfo.tcl
+} elseif [file exists conninfo.tcl] {
+    puts [list sourcing conninfo.tcl ...]
+    source conninfo.tcl
+}
+
+puts [list preparing the database]
+set conn [pg::connect -connlist [array get ::conninfo]]
+set res [$conn exec {create table big__test__table (c0 varchar primary key, c1 varchar, c2 varchar, c3 varchar)}]
+if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	# Check if it's a real error or a left-over table
+	set expected "ERROR:  relation \"big__test__table\" already exists\n"
+	set actual [pg_result $res -error]
+	pg_result $res -clear
+	if {$actual ne $expected} {
+		puts $actual
+		exit -1
+	}
+	# Check if the table has the expected columns
+	set result ""
+	set expected [list c0 c1 c2 c3]
+	pg_select $conn {select * from big__test__table LIMIT 1;} row {
+		set result $row(.headers)
+	}
+	if {$result ne $expected} {
+		puts [list Expected columns $expected got $result]
+		exit -1
+	}
+} else {
+	pg_result $res -clear
+}
+pg_disconnect $conn
+
+puts [list running tests ...]
+if 0 {
+#
+#
+#
+test pgtcl-12.1 {Fill table with random data using paramarray} -body {
+    unset -nocomplain res
+
+    set conn [pg::connect -connlist [array get ::conninfo]]
+    set res [$conn exec "delete from big__test__table;"]
+    if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	return [pg_result $res -error]
+    }
+    pg_result $res -clear
+
+    for {set i 0} {$i < 10000} {incr i} {
+	set array(c0) $i
+	set array(c1) [expr rand()]
+	set array(c2) [expr rand()]
+	set array(c3) [expr rand()]
+        set res [$conn exec -paramarray array {insert into big__test__table (c0, c1, c2, c3) values (`c0`, `c1`, `c2`, `c3`)}]
+        if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+        }
+	pg_result $res -clear
+	puts -nonewline stderr "1.1 $i       \r"
+    }
+    for {set i 0} {$i < 10000} {incr i} {
+	set array(c0) [expr {int(rand() * 10000)}]
+	set array(c1) [expr rand()]
+	set array(c2) [expr rand()]
+	set array(c3) [expr rand()]
+        set res [$conn exec -paramarray array {update big__test__table set c1 = `c1`, c2 = `c2`, c3=`c3` where c0 = `c0`;}]
+        if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+        }
+	pg_result $res -clear
+	puts -nonewline stderr "1.2 $i       \r"
+    }
+
+    pg_disconnect $conn
+
+    list
+} -result [list]
+
+#
+#
+#
+test pgtcl-12.2 {Fill table with random data using inline values} -body {
+    unset -nocomplain res
+
+    set conn [pg::connect -connlist [array get ::conninfo]]
+    set res [$conn exec "delete from big__test__table;"]
+    if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	return [pg_result $res -error]
+    }
+    pg_result $res -clear
+
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 $i
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec {insert into big__test__table (c0, c1, c2, c3) values ($1, $2, $3, $4)} $c0 $c1 $c2 $c3]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "2.1 $i       \r"
+    }
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 [expr {int(rand() * 10000)}]
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec {update big__test__table set c1 = $2, c2 = $3, c3=$4 where c0 = $1;} $c0 $c1 $c2 $c3]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "2.2 $i       \r"
+    }
+
+    pg_disconnect $conn
+
+    list
+} -result [list]
+
+#
+#
+#
+test pgtcl-12.3 {Fill table with random data using variables} -body {
+    unset -nocomplain res
+
+    set conn [pg::connect -connlist [array get ::conninfo]]
+    set res [$conn exec "delete from big__test__table;"]
+    if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	return [pg_result $res -error]
+    }
+    pg_result $res -clear
+
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 $i
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec -variables {insert into big__test__table (c0, c1, c2, c3) values (:c0, :c1, :c2, :c3)}]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "3.1 $i       \r"
+    }
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 [expr {int(rand() * 10000)}]
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec -variables {update big__test__table set c1 = :c1, c2 = :c2, c3=:c3 where c0 = :c0;}]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "3.2 $i       \r"
+    }
+    pg_disconnect $conn
+
+    list
+} -result [list]
+
+#
+#
+#
+test pgtcl-13.1 {pg_select with -variables} -body {
+    unset -nocomplain res
+
+    set conn [pg::connect -connlist [array get ::conninfo]]
+    set res [$conn exec "delete from big__test__table;"]
+    if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	return [pg_result $res -error]
+    }
+    pg_result $res -clear
+
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 $i
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec -variables {insert into big__test__table (c0, c1, c2, c3) values (:c0, :c1, :c2, :c3)}]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "13.1.1 $i       \r"
+    }
+    set result 0
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 [expr {int(rand() * 10000)}]
+        $conn select -variables {SELECT * FROM big__test__table WHERE c0 = :c0;} row {
+		incr result
+	}
+	puts -nonewline stderr "13.1.2 $i       \r"
+    }
+    puts stderr [list result $result]
+    pg_disconnect $conn
+
+    list
+} -result [list]
+
+#
+#
+#
+test pgtcl-13.2 {pg_select with -paramarray} -body {
+    unset -nocomplain res
+
+    set conn [pg::connect -connlist [array get ::conninfo]]
+    set res [$conn exec "delete from big__test__table;"]
+    if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	return [pg_result $res -error]
+    }
+    pg_result $res -clear
+
+    for {set i 0} {$i < 10000} {incr i} {
+	set array(c0) $i
+	set array(c1) [expr rand()]
+	set array(c2) [expr rand()]
+	set array(c3) [expr rand()]
+        set res [$conn exec -paramarray array {insert into big__test__table (c0, c1, c2, c3) values (`c0`, `c1`, `c2`, `c3`)}]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "13.1.1 $i       \r"
+    }
+    set result 0
+    for {set i 0} {$i < 10000} {incr i} {
+	set array(c0) [expr {int(rand() * 10000)}]
+        $conn select -paramarray array {SELECT * FROM big__test__table WHERE c0 = `c0`;} row {
+		incr result
+	}
+	puts -nonewline stderr "13.1.2 $i       \r"
+    }
+    puts stderr [list result $result]
+    pg_disconnect $conn
+
+    list
+} -result [list]
+
+#
+#
+#
+test pgtcl-13.3 {pg_select with -params} -body {
+    unset -nocomplain res
+
+    set conn [pg::connect -connlist [array get ::conninfo]]
+    set res [$conn exec "delete from big__test__table;"]
+    if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	return [pg_result $res -error]
+    }
+    pg_result $res -clear
+
+    puts "\rpgtcl-13.3            "
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 $i
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec {insert into big__test__table (c0, c1, c2, c3) values ($1, $2, $3, $4)} $c0 $c1 $c2 $c3]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "\r13.1.1 $i       "
+    }
+    set result 0
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 [expr {int(rand() * 10000)}]
+        $conn select -params [list $c0] {SELECT * FROM big__test__table WHERE c0 = $1;} row {
+		incr result
+	}
+	puts -nonewline stderr "\r13.1.2 $i       "
+    }
+    puts stderr "\r[list result $result]         "
+    pg_disconnect $conn
+
+    list
+} -result [list]
+}
+#
+#
+#
+test pgtcl-14.1 {comprehensive tests} -body {
+    unset -nocomplain res
+
+    set conn [pg::connect -connlist [array get ::conninfo]]
+    set res [$conn exec "delete from big__test__table;"]
+    if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	return [pg_result $res -error]
+    }
+    pg_result $res -clear
+
+    puts "\rpgtcl-14.1            "
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 $i
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec {insert into big__test__table (c0, c1, c2, c3) values ($1, $2, $3, $4)} $c0 $c1 $c2 $c3]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "\r14.1.1 $i       "
+    }
+    set result 0
+    for {set i 0} {$i < 10000} {incr i} {
+	set c0 [expr {int(rand() * 10000)}]
+        $conn select -variables {SELECT * FROM big__test__table WHERE c0 = :c0;} row {
+		incr result
+	}
+	set c1 [expr rand()]
+	set c2 [expr rand()]
+	set c3 [expr rand()]
+        set res [$conn exec {update big__test__table set c1 = $2, c2 = $3, c3=$4 where c0 = $1;} $c0 $c1 $c2 $c3]
+	if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts ""
+	    return [pg_result $res -error]
+	}
+	pg_result $res -clear
+	set c0 [expr {int(rand() * 10000)}]
+        $conn select -params [list $c0] {SELECT * FROM big__test__table WHERE c0 = $1;} row {
+		incr result
+	}
+	set array(c0) [expr {int(rand() * 10000)}]
+	set array(c1) [expr rand()]
+	set array(c2) [expr rand()]
+	set array(c3) [expr rand()]
+        set res [$conn exec -paramarray array {update big__test__table set c1 = `c1`, c2 = `c2`, c3=`c3` where c0 = `c0`;}]
+        if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+	    puts "{update big__test__table set c1 = `c1`, c2 = `c2`, c3=`c3` where c0 = `c0`;}" failed
+	    return [pg_result $res -error]
+        }
+	pg_result $res -clear
+        $conn select -paramarray array {SELECT * FROM big__test__table WHERE c0 = `c0`;} row {
+		incr result
+	}
+	pg_execute $conn "select * from big__test__table WHERE c0 = [pg_quote $c0]" {
+		incr result
+	}
+        pg_select -paramarray array $conn {SELECT * FROM big__test__table WHERE c0 = `c0`;} row {
+		incr result
+	}
+        set res [$conn exec -variables {SELECT * FROM big__test__table WHERE c0 = :c0}]
+	set stat [pg_result $res -status]
+        if {$stat ne "PGRES_COMMAND_OK" && $stat ne "PGRES_TUPLES_OK"} {
+	    set err [pg_result $res -error]
+	    puts "{SELECT * FROM big__test__table WHERE c0 = :c0} status [pg_result $res -status] error  $err"
+	    return $err
+        }
+	pg_result $res -foreach row {
+		incr result
+	}
+	pg_result $res -clear
+	puts -nonewline stderr "\r14.1.2 $i $result      "
+    }
+    puts stderr "\r[list result $result]         "
+    pg_disconnect $conn
+
+    list
+} -result [list]
+
+puts "cleaning up"
+set conn [pg::connect -connlist [array get ::conninfo]]
+set res [$conn exec "drop table big__test__table;"]
+if {[pg_result $res -status] ne "PGRES_COMMAND_OK"} {
+    return [pg_result $res -error]
+}
+pg_result $res -clear
+pg_disconnect $conn
+
+
+puts "tests complete"

From 3039042237d127f2ad3c603b8f1412692e972e9e Mon Sep 17 00:00:00 2001
From: Peter da Silva <peter.dasilva@flightaware.com>
Date: Mon, 6 May 2024 15:20:07 +0000
Subject: [PATCH 14/15] Update minor version number.

---
 configure.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.in b/configure.in
index 0e093fe..a61ecd0 100755
--- a/configure.in
+++ b/configure.in
@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so you can encode the package version directly into the source files.
 #-----------------------------------------------------------------------
 
-AC_INIT([pgtcl], [3.0.1])
+AC_INIT([pgtcl], [3.1.0])
 
 #-----
 # Version with patch stripped

From 33587e8e8d214bc5e1e08ccc6c9bc33b1e976374 Mon Sep 17 00:00:00 2001
From: Jeff Lawson <jeff.lawson@flightaware.com>
Date: Sat, 11 May 2024 00:06:05 +0000
Subject: [PATCH 15/15] update mac ci for modern brew

---
 .github/workflows/mac-ci.yml | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/mac-ci.yml b/.github/workflows/mac-ci.yml
index 4569381..72cb6c3 100644
--- a/.github/workflows/mac-ci.yml
+++ b/.github/workflows/mac-ci.yml
@@ -16,18 +16,14 @@ jobs:
     - name: Install dependencies
       run: |
         brew update
-        brew install tcl-tk || true
-        sudo mkdir -p /usr/local
-        sudo ln -sf /usr/local/opt/tcl-tk/include /usr/local/include/tcl8.6
-        sudo rm -f /usr/local/lib/libtcl* || true
-        sudo cp /usr/local/opt/tcl-tk/lib/libtcl* /usr/local/lib
-        sudo ln -sf /usr/local/opt/tcl-tk/bin/tclsh8.6 /usr/local/bin/tclsh
-        sudo ln -sf /usr/local/opt/tcl-tk/bin/tclsh8.6 /usr/local/bin/tclsh8.6
+        brew install tcl-tk autoconf libpq
     - name: configure
       run: |       
         autoreconf -vi
-        ./configure --with-tcl=/usr/local/opt/tcl-tk/lib --prefix=/usr/local        
+        export PG_CONFIG=/opt/homebrew/opt/libpq/bin/pg_config
+        ./configure --with-tcl=/opt/homebrew/opt/tcl-tk/lib/ --prefix=/usr/local        
     - name: make
       run: make
     - name: install
       run: sudo make install
+