Schema now does not contain a copy of the entire table (#659)

* .stpps.schemas now contains only schemas and no other data
* Preserving schema format for partitioned DBs
* Added test case

---------

Co-authored-by: GGabi <[email protected]>

Author: GGabi

code/common/pubsub.q

@@ -114,7 +114,7 @@ attrstrip:{[t]
 init:{[t]
   if[count b:t where not t in tables[];{.lg.e[`psinit;m:"Table ",string[x]," does not exist"];'m} each b];
   .stpps.t:t except b;
-  .stpps.schemas:.stpps.t!value each .stpps.t;
+  .stpps.schemas:.stpps.t!{t: value x; $[.Q.qp t; t; 0#t]} each .stpps.t;
   .stpps.tabcols:.stpps.t!cols each .stpps.t;
  };
 

tests/schemas/insert_data.q

@@ -0,0 +1,7 @@
+stpHandle:gethandle[`stp1]
+wdbHandle:gethandle[`wdb1]
+testtrade:((5#`GOOG),5?`4;10?100.0;10?100i;10#0b;10?.Q.A;10?.Q.A;10#`buy)
+testquote:(10?`4;(5?50.0),50+5?50.0;10?100.0;10?100i;10?100i;10?.Q.A;10?.Q.A;10#`3)
+stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))
+wdbHandle(`.u.end;`.wdb.currentpartition)
+exit 0

tests/schemas/process.csv

@@ -0,0 +1,6 @@
+host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd
+localhost,{KDBBASEPORT}+100,discovery,discovery1,,1,0,,,${KDBCODE}/processes/discovery.q,1,,q
+localhost,{KDBBASEPORT}+101,segmentedtickerplant,stp1,,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQHOME}/database.q,q
+localhost,{KDBBASEPORT}+103,rdb,rdb1,,1,1,180,,${KDBCODE}/processes/rdb.q,1,,q
+localhost,{KDBBASEPORT}+104,hdb,hdb1,,1,1,180,,${TORQHOME}/hdb,1,,q
+localhost,{KDBBASEPORT}+105,wdb,wdb1,,1,1,180,,${KDBCODE}/processes/wdb.q,1,,q

tests/schemas/run.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Handle command-line arguments
+source $KDBTESTS/flagparse.sh
+
+#Set up the test env with data
+#path to test directory
+testpath=${KDBTESTS}/schemas
+
+#start procs
+${TORQHOME}/torq.sh start discovery1 rdb1 stp1 hdb1 wdb1 -csv ${testpath}/process.csv
+${TORQHOME}/torq.sh summary -csv ${testpath}/process.csv
+
+#Start test proc
+/usr/bin/rlwrap q ${TORQHOME}/torq.q \
+	-proctype test -procname test1 \
+	-load ${KDBTESTS}/helperfunctions.q ${testpath}/settings.q ${testpath}/insert_data.q     \
+	-procfile ${testpath}/process.csv -quiet
+
+${TORQHOME}/torq.sh stop all -csv ${testpath}/process.csv
+
+# Do the tests
+#start procs
+${TORQHOME}/torq.sh start discovery1 rdb1 stp1 hdb1 wdb1 -csv ${testpath}/process.csv
+
+#Start test proc
+/usr/bin/rlwrap q ${TORQHOME}/torq.q \
+	-proctype test -procname test1 \
+	-test ${testpath} \
+	-load ${KDBTESTS}/helperfunctions.q ${testpath}/settings.q     \
+	-procfile ${testpath}/process.csv -debug
+
+#Shut down procs
+${TORQHOME}/torq.sh stop all -csv ${testpath}/process.csv

tests/schemas/settings.q

@@ -0,0 +1,4 @@
+.servers.CONNECTIONS:`wdb`segmentedtickerplant`hdb`idb`gateway`rdb
+.servers.startup[];
+
+hdbdir: `$getenv[`TORQHOME],"/wdbhdb/"

tests/schemas/test.csv

@@ -0,0 +1,17 @@
+action,ms,bytes,lang,code,repeat,minver,comment
+comment,0,,"Test 1 - Check all the schemas are empty after having written to the DB"
+before,0,0,q,\c 200 200,1,,"Set width of .Q.s1 to be wide enough for all tables"
+before,0,0,q,.servers.startup[],1,,"Start TorQ Connection Management"
+before,0,0,q,.proc.sys "sleep 2",1,,"Wait for process to start"
+before,0,0,q,hdbHandle:gethandle[`hdb1],1,,"Open HDB handle"
+before,0,0,q,schemas:hdbHandle`.stpps.schemas,1,,"Open HDB handle"
+true,0,0,q,([]time:`timestamp$();sym:`symbol$();procname:`symbol$();counter:`long$();pid:`int$();host:`symbol$();port:`int$())~schemas`heartbeat,,,"Check the schema is empty"
+true,0,0,q,([]time:`timestamp$();sym:`symbol$();proctype:`symbol$();host:`symbol$();loglevel:`symbol$();id:`symbol$();message:())~schemas`logmsg,,,"Check the schema is empty"
+true,0,0,q,""+`time`sym`src`dest`srcport`destport`seq`ack`win`tsval`tsecr`flags`protocol`length`len`data!`packets"~.Q.s1 schemas`packets,,,"Check the partitioned table schema is as expected"
+true,0,0,q,""+`time`sym`bid`ask`bsize`asize`mode`ex`src!`quote"~.Q.s1 schemas`quote,,,"Check the partitioned table schema is as expected"
+true,0,0,q,""+`time`sym`bid`ask`bsize`asize`mode`ex`srctime!`quote_iex"~.Q.s1 schemas`quote_iex,,,"Check the partitioned table schema is as expected"
+true,0,0,q,""+`time`sym`price`size`stop`cond`ex`side!`trade"~.Q.s1 schemas`trade,,,"Check the partitioned table schema is as expected"
+true,0,0,q,""+`time`sym`price`size`stop`cond`ex`srctime!`trade_iex"~.Q.s1 schemas`trade_iex,,,"Check the partitioned table schema is as expected"
+after,0,0,q,hclose hdbHandle,1,,"Close handles"
+after,0,0,q,.os.deldir logdir,1,,"Delete test logs"
+after,0,0,q,.os.deldir hdbdir,1,,"Delete test hdb"