From b74ad0e7ebf2394cb2c59c08d923f2d06b92182a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferenc=20Heged=C5=B1s?= <145510059+Papamaci444@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:51:38 +0100 Subject: [PATCH] new writedown mode partbyenum and new intraday process (#656) * added partbyenum writedown mode * new idb process * added idb process * changing EOL to unix * changed idb logic * idb notification logging refined * createalias refined and added to os.q * addressed comments, removed gmttime, refactored * correcting indentation, removing reload function from idbstandard.q * removing unused server types from wdb * added back comment in processes/wdb.q * added hsym to folder parameters * added testing wdb partbyenum * added test for idb process, added hsym to hdbdir and wdbdir parmters for idb * added documentation for partbyenum and idb * added graphics for idb docs * EOL changed to LF * IDB is accessible via Gateway * IDB doesn't fail on restart or empty database * IDB reworked - connects to WDB and registers itself with WDB. IDB has no significant downtime now. WDB now initialises the DB after EOD rollover. * WDB refactored to support new writedown mode partbyenum more * IDB intraday reload and logging improved. Fixed some comments. --- code/common/merge.q | 10 +- code/common/os.q | 2 +- code/processes/idb.q | 95 ++++ code/processes/wdb.q | 575 +++++++++++++----------- code/wdb/origstartup.q | 34 +- code/wdb/writedown.q | 14 +- config/passwords/idb.txt | 1 + config/settings/gateway.q | 14 +- config/settings/idb.q | 11 + config/settings/sort.q | 17 +- config/settings/wdb.q | 12 +- docs/Processes.md | 45 +- docs/graphics/idb.png | Bin 0 -> 40780 bytes tests/stp/idbdefault/config/sort.csv | 4 + tests/stp/idbdefault/idbdefault.csv | 35 ++ tests/stp/idbdefault/process.csv | 8 + tests/stp/idbdefault/run.sh | 23 + tests/stp/idbdefault/settings.q | 17 + tests/stp/idbpartbyenum/config/sort.csv | 4 + tests/stp/idbpartbyenum/idbenum.csv | 40 ++ tests/stp/idbpartbyenum/process.csv | 9 + tests/stp/idbpartbyenum/run.sh | 23 + tests/stp/idbpartbyenum/settings.q | 17 + tests/stp/wdb/config/sort.csv | 4 + tests/stp/wdb/partbyenum.csv | 32 ++ tests/stp/wdb/process.csv | 2 + tests/stp/wdb/settings.q | 11 +- 27 files changed, 754 insertions(+), 305 deletions(-) create mode 100644 code/processes/idb.q create mode 100644 config/passwords/idb.txt create mode 100644 config/settings/idb.q create mode 100644 docs/graphics/idb.png create mode 100644 tests/stp/idbdefault/config/sort.csv create mode 100644 tests/stp/idbdefault/idbdefault.csv create mode 100644 tests/stp/idbdefault/process.csv create mode 100644 tests/stp/idbdefault/run.sh create mode 100644 tests/stp/idbdefault/settings.q create mode 100644 tests/stp/idbpartbyenum/config/sort.csv create mode 100644 tests/stp/idbpartbyenum/idbenum.csv create mode 100644 tests/stp/idbpartbyenum/process.csv create mode 100644 tests/stp/idbpartbyenum/run.sh create mode 100644 tests/stp/idbpartbyenum/settings.q create mode 100644 tests/stp/wdb/config/sort.csv create mode 100644 tests/stp/wdb/partbyenum.csv diff --git a/code/common/merge.q b/code/common/merge.q index df7fef9d3..e10a9a844 100644 --- a/code/common/merge.q +++ b/code/common/merge.q @@ -20,13 +20,19 @@ getextrapartitiontype:{[tablename] tabparts }; -/- function to check each partiton type specified in sort.csv is actually present in specified table +/- function to check each partition type specified in sort.csv is actually present in specified table checkpartitiontype:{[tablename;extrapartitiontype] $[count colsnotintab:extrapartitiontype where not extrapartitiontype in cols get tablename; .lg.e[`checkpart;"parted columns ",(", " sv string colsnotintab)," are defined in sort.csv but not present in ",(string tablename)," table"]; .lg.o[`checkpart;"all parted columns defined in sort.csv are present in ",(string tablename)," table"]]; }; +/- function to check if the extra partition column has a symbol type +checksymboltype:{[tablename;extrapartitiontype] + $[all extrapartitiontype in exec c from meta[tablename] where t="s"; + .lg.o[`checksymbol;"all columns do have a symbol type in ",(string tablename)," table"]; + .lg.e[`checksymbol;"not all columns ",string[extrapartitiontype]," do have a symbol type in ",(string tablename)," table"]]; + }; /- function to get list of distinct combiniations for partition directories @@ -66,7 +72,7 @@ mergebypart:{[tablename;dest;partchunks] .lg.o[`merge;"upserting ",(string count chunks)," rows to ",string dest]; /-merge columns to permanent storage .[upsert;(dest;chunks); - {.lg.e[`merge;"failed to merge to ", sting[dest], " from segments ", (", " sv string chunks)];}]; + {.lg.e[`merge;"failed to merge to ", string[dest], " from segments ", (", " sv string chunks)];}]; }; /-merge data from partition in temporary storage to permanent storage, column by column rather than by entire partition diff --git a/code/common/os.q b/code/common/os.q index 9853115e1..db6ff4f20 100644 --- a/code/common/os.q +++ b/code/common/os.q @@ -21,4 +21,4 @@ df:{(`$("/";"\\")[NT]sv -1_v;`$-1#v:("/";"\\")[NT]vs pth(string x;x)[10h=type x] run:{system"q ",x} kill:{[p]@[(`::p);"\\\\";1];} sleep:{x:string x; system("sleep ",x;"timeout /t ",x," >nul")[NT]} -pthq:{[x] $[10h=type x;ssr [x;"\\";"/"];`$ -1 _ ssr [string (` sv x,`);"\\";"/"]]} +pthq:{[x] $[10h=type x;ssr [x;"\\";"/"];`$ -1 _ ssr [string (` sv x,`);"\\";"/"]]} \ No newline at end of file diff --git a/code/processes/idb.q b/code/processes/idb.q new file mode 100644 index 000000000..a663b7747 --- /dev/null +++ b/code/processes/idb.q @@ -0,0 +1,95 @@ +/-default parameters +\d .idb + +wdbtypes:@[value;wdbtypes;`wdb]; + +/-these parameters are only used once their value has been set with values retrieved from the WBD. +writedownmode:idbdir:savedir:currentpartition:symfilepath:`; +symsize:partitionsize:0; + +/-force loads sym file +loadsym:{[] + .lg.o[`load;"loading the sym file"]; + @[load;symfilepath; {.lg.e[`load;"failed to load sym file: ",string[symfilepath]," error: ",x]}]; + symsize::hcount symfilepath; + }; + +/-force loads IDB +loadidb:{[] + .lg.o[`load;"loading the db"]; + @[system; "l ", 1_string idbdir; {.lg.e[`load;"failed to load IDB: ",string[idbdir]," error: ",x]}]; + partitionsize::count key idbdir; + }; + +/- force loads the idb and the sym file +loaddb:{[] + starttime:.proc.ct[]; + loadsym[]; + loadidb[]; + .lg.o[`load;"IDB load has been finished for partition: ",string[currentpartition],". Time taken(ms): ",string .proc.ct[]-starttime]; + }; + +/- sets current partition and force loads the idb and the sym file. Called by the WDB after EOD. +rollover:{[pt] + currentpartition::pt; + idbdir::.Q.dd[savedir; currentpartition]; + loaddb[]; + }; + +/- reloads the db. Called by wdb process midday/eod. +intradayreload:{[] + starttime:.proc.ct[]; + if[symfilehaschanged[];loadsym[]]; + if[partitioncounthaschanged[];loadidb[]]; + .lg.o[`intradayreload;"IDB reload has been finished for partition: ",string[savedir],". Time taken(ms): ",string .proc.ct[]-starttime]; + }; + +/- checks if sym file has changed since last reload of the IDB. Records new sym size if changed. +symfilehaschanged:{[] + $[symsize<>c:hcount symfilepath;[symsize::c; 1b];0b] + }; + +/- checks if count of partitions has changed since last reload of the IDB. Records new partition count if changed. +/- the default writedown method doesn't need db reloading as no new directory is being created there. +partitioncounthaschanged:{[] + if[writedownmode~`default;:0b]; + $[partitionsize<>c:count key idbdir;[partitionsize::c; 1b];0b] + }; + +setparametersfromwdb:{[wdbHandle] + .lg.o[`init;"querying WDB, HDB locations, current partition and writedown mode from WDB"]; + params:@[wdbHandle; (each;value;`.wdb.savedir`.wdb.hdbdir`.wdb.currentpartition`.wdb.writedownmode); {.lg.e[`connection; "Failed to retrieve values from WDB."]; 'x}]; + savedir::hsym params[0]; + currentpartition::params[2]; + symfilepath::.Q.dd[hsym params[1]; `sym]; + writedownmode::params[3]; + idbdir::.Q.dd[savedir; $[writedownmode~`default;`;currentpartition]]; + .lg.o[`init;"Current settings: db folder: ",string[idbdir],", sym file: ",string[symfilepath],", writedownmode: ", string writedownmode]; + }; + +init:{[] + .lg.o[`init; "searching for servers"]; + .servers.startup[]; + .lg.o[`init;"getting connection handle to the WDB"]; + w:.servers.gethandlebytype[wdbtypes;`any]; + /-exit if no valid handle + if[0=count w; .lg.e[`connection;"no connection to the WDB could be established... failed to initialise."];:()]; + .lg.o[`init;"found a WDB process"]; + /-setting parameters in .idb namespace from WDB + setparametersfromwdb[w]; + .lg.o[`init;"loading the db and the sym file first time"]; + loaddb[]; + .lg.o[`init;"registering IDBs on WDB process..."]; + /-send sync message to WDB to register the existing IDBs. + @[w;(`.servers.registerfromdiscovery;`idb;0b);{.lg.e[`connection;"Failed to register IDB with WDB."];'x}]; + .lg.o[`init; "Initialisation of the IDB is done."]; + } + +\d . + +.idb.init[]; + +/- helper function to support queries against the sym column +maptoint:{[symbol] + sym?symbol + }; diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 1f52cda67..b4a84d841 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -1,4 +1,4 @@ -/-TorQ wdb process - based upon w.q +/-TorQ wdb process - based upon w.q /http://code.kx.com/wsvn/code/contrib/simon/tick/w.q /-subscribes to tickerplant and appends data to disk after the in-memory table exceeds a specified number of rows /-the row check is set on a timer - the interval may be specified by the user @@ -11,7 +11,7 @@ /- define default parameters mode:@[value;`mode;`saveandsort]; /-the wdb process can operate in three modes /- 1. saveandsort - the process will subscribe for data, - /- periodically write data to disk and at EOD it will flush + /- periodically write data to disk and at EOD it will flush /- remaining data to disk before sorting it and informing /- GWs, RDBs and HDBs etc... /- 2. save - the process will subscribe for data, @@ -28,11 +28,14 @@ writedownmode:@[value;`writedownmode;`default]; /-the /- at EOD the data will be sorted and given attributes according to sort.csv before being moved to hdb /- 2. partbyattr - the data is partitioned by [ partitiontype ] and the column(s) assigned the parted attributed in sort.csv /- at EOD the data will be merged from each partiton before being moved to hdb + /- 3. partbyenum - the data is partitioned by [ partitiontype ] and a symbol column with parted attribution assigned in sort.csv + /- at EOD the data will be merged from each partiton before being moved to hdb +enumcol:@[value;`enumcol;`sym]; /-symbol column to enumerate by. Only used with writedownmode: partbyenum. -mergemode:@[value;`mergemode;`part]; /-the partbyattr writdown mode can merge data from tenmporary storage to the hdb in three ways: - /- 1. part - the entire partition is merged to the hdb - /- 2. col - each column in the temporary partitions are merged individually - /- 3. hybrid - partitions merged by column or entire partittion based on byte limit +mergemode:@[value;`mergemode;`part]; /-the partbyattr writdown mode can merge data from tenmporary storage to the hdb in three ways: + /- 1. part - the entire partition is merged to the hdb + /- 2. col - each column in the temporary partitions are merged individually + /- 3. hybrid - partitions merged by column or entire partittion based on byte limit mergenumbytes:@[value;`mergenumbytes;500000000]; /-default number of bytes for merge process @@ -41,12 +44,13 @@ mergenumtab:@[value;`mergenumtab;`quote`trade!10000 50000]; /-spe hdbtypes:@[value;`hdbtypes;`hdb]; /-list of hdb types to look for and call in hdb reload rdbtypes:@[value;`rdbtypes;`rdb]; /-list of rdb types to look for and call in rdb reload +idbtypes:@[value;`idbtypes;`idb]; /-list of idb types to look for and call in idb reload gatewaytypes:@[value;`gatewaytypes;`gateway]; /-list of gateway types to inform at reload tickerplanttypes:@[value;`tickerplanttypes;`tickerplant]; /-list of tickerplant types to try and make a connection to tpconnsleepintv:@[value;`tpconnsleepintv;10]; /-number of seconds between attempts to connect to the tp -tpcheckcycles:@[value;`tpcheckcycles;0W]; /-number of attempts to connect to tp before process is killed +tpcheckcycles:@[value;`tpcheckcycles;0W]; /-number of attempts to connect to tp before process is killed -sorttypes:@[value;`sorttypes;`sort]; /-list of sort types to look for upon a sort +sorttypes:@[value;`sorttypes;`sort]; /-list of sort types to look for upon a sort sortworkertypes:@[value;`sortworkertypes;`sortworker]; /-list of sort types to look for upon a sort being called with worker process subtabs:@[value;`subtabs;`]; /-list of tables to subscribe for @@ -58,7 +62,7 @@ replay:@[value;`replay;1b]; /-rep schema:@[value;`schema;1b]; /-retrieve schema from tickerplant settimer:@[value;`settimer;0D00:00:10]; /-set timer interval for row check -reloadorder:@[value;`reloadorder;`hdb`rdb]; /-order to reload hdbs and rdbs +reloadorder:@[value;`reloadorder;`hdb`rdb]; /-order to reload hdbs, rdbs sortcsv:@[value;`sortcsv;`:config/sort.csv]; /-location of csv file permitreload:@[value;`permitreload;1b]; /-enable reload of hdbs/rdbs @@ -74,8 +78,8 @@ eodwaittime:@[value;`eodwaittime;0D00:00:10.000]; /-len / - define .z.pd in order to connect to any worker processes .dotz.set[`.z.pd;{$[.z.K<3.3; - `u#`int$(); - `u#exec w from .servers.getservers[`proctype;sortworkertypes;()!();1b;0b]]}] + `u#`int$(); + `u#exec w from .servers.getservers[`proctype;sortworkertypes;()!();1b;0b]]}] /- fix any backslashes on windows savedir:.os.pthq savedir; @@ -85,6 +89,9 @@ hdbdir:.os.pthq hdbdir; saveenabled: any `save`saveandsort in mode; sortenabled: any `sort`saveandsort in mode; +/- parted writedown modes have special behaviour during merging or WDB initialisation +partwritemodes:`partbyattr`partbyenum; + / - log which modes are enabled switch: string `off`on; .lg.o[`savemode;"save mode is ",switch[saveenabled]]; @@ -101,150 +108,175 @@ mergemaxrows:{[tabname] mergenumrows^mergenumtab[tabname]} /- function to return a list of tables that the wdb process has been configured to deal within -tablelist:{[] sortedlist:exec tablename from `bytes xdesc .wdb.tabsizes; - (sortedlist union tables[`.]) except ignorelist} +tablelist:{[] sortedlist:exec tablename from `bytes xdesc tabsizes; + (sortedlist union tables[`.]) except ignorelist} /- function to upsert to specified directory -upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt] - .lg.o[`save;"saving ",(string tablename)," data to partition ", - /- create directory location for selected partiton - string directory:` sv .Q.par[dir;pt;tablename], - /- replace random chracters in symbols with _ - (`$"_"^.Q.an .Q.an?"_" sv string - /- convert to symbols and replace any null values with `NONE - `NONE^ -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} expt,(::)),`]; - /- upsert selected data matched on partition to specific directory - .[ - upsert; - (directory;r:?[tabdata;{(x;y;(),z)}[in;;]'[expttype;expt];0b;()]); - {[e] .lg.e[`savetablesbypart;"Failed to save table to disk : ",e];'e} - ]; - .lg.o[`track;"appending details to partsizes"]; - /-key in partsizes are directory to partition, need to drop training slash in directory key - .merge.partsizes[first ` vs directory]+:(count r;-22!r); - }; - -savetablesbypart:{[dir;pt;forcesave;tablename] - /- check row count and save if maxrows exceeded - /- forcesave will write flush the data to disk irrespective of counts - if[forcesave or maxrows[tablename] < arows: count value tablename; - .lg.o[`rowcheck;"the ",(string tablename)," table consists of ", (string arows), " rows"]; - /- get additional partition(s) defined by parted attribute in sort.csv - extrapartitiontype:.merge.getextrapartitiontype[tablename]; - /- check each partition type actually is a column in the selected table - .merge.checkpartitiontype[tablename;extrapartitiontype]; - /- get list of distinct combiniations for partition directories - extrapartitions:.merge.getextrapartitions[tablename;extrapartitiontype]; - /- enumerate data to be upserted - enumdata:.Q.en[hdbsettings[`hdbdir];0!.save.manipulate[tablename;`. tablename]]; - .lg.o[`save;"enumerated ",(string tablename)," table"]; - /- upsert data to specific partition directory - upserttopartition[dir;tablename;enumdata;pt;extrapartitiontype] each extrapartitions; - /- empty the table - .lg.o[`delete;"deleting ",(string tablename)," data from in-memory table"]; - @[`.;tablename;0#]; - /- run a garbage collection (if enabled) - if[gc;.gc.run[]]; - ]; - }; - +upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] + /- enumerate current extra partition against the hdb sym file + if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)?first expt;]; + /- create directory location for selected partiton + /- replace non-alphanumeric characters in symbols with _ + /- convert to symbols and replace any null values with `NONE + directory:$[writedownmode~`partbyenum; + ` sv .Q.par[dir;pt;`$string i],tablename,`; + ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `NONE^ -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} expt,(::)),`]; + .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory]; + /- selecting rows of table with matching partition + r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; + /- upsert selected data matched on partition to specific directory + .[upsert;(directory;r);{[e] .lg.e[`savetablesbypart;"Failed to save table to disk : ",e];'e}]; + .lg.o[`track;"appending details to partsizes"]; + /-key in partsizes are directory to partition, need to drop training slash in directory key + .merge.partsizes[first ` vs directory]+:(count r;-22!r); + }; + +savetablesbypart:{[dir;pt;forcesave;tablename;writedownmode] + /- check row count and save if maxrows exceeded + /- forcesave will write flush the data to disk irrespective of counts + if[forcesave or maxrows[tablename] < arows: count value tablename; + .lg.o[`rowcheck;"the ",(string tablename)," table consists of ", (string arows), " rows"]; + /- get additional partition(s) defined by parted attribute in sort.csv + extrapartitiontype:.merge.getextrapartitiontype[tablename]; + if[(writedownmode~`partbyenum) and 1.wdb.timeouttime) or (count[.wdb.reloadsummary]=.wdb.countreload); + /-insert process reload outcome into reloadsummary + reloadsummary[.z.w]:x; + /-log result of reload in wdb out log + .lg.o[`reloadproc;"the ", string[reloadsummary[.z.w]`process]," process ", string[reloadsummary[.z.w]`result]]; + if[(.proc.cp[]>timeouttime) or (count[reloadsummary]=countreload); .lg.o[`handler;"releasing processes"]; - .lg.o[`reload;string[count select from .wdb.reloadsummary where status=1]," out of ", string[count .wdb.reloadsummary]," processes successfully reloaded"]; - .wdb.flushend[]; - /-delete contents from .wdb.reloadsummary when reloads completed + .lg.o[`reload;string[count select from reloadsummary where status=1]," out of ", string[count reloadsummary]," processes successfully reloaded"]; + flushend[]; + /-delete contents from reloadsummary when reloads completed delete from `.wdb.reloadsummary]; - }; + }; /- evaluate contents of d dictionary asynchronously /- notify the gateway that we are done flushend:{ - if[not @[value;`.wdb.reloadcomplete;0b]; - @[{neg[x]"";neg[x][]};;()] each key reloadsummary; - informgateway(`reloadend;`); - .lg.o[`sort;"end of day sort is now complete"]; - .wdb.reloadcomplete:1b]; - /- run a garbage collection (if enabled) - if[gc;.gc.run[]]; - }; - -/- initialise reloadsummary, keyed tale to track status of local reloads + if[not @[value;`.wdb.reloadcomplete;0b]; + @[{neg[x]"";neg[x][]};;()] each key reloadsummary; + informgateway(`reloadend;`); + .lg.o[`sort;"end of day sort is now complete"]; + reloadcomplete::1b]; + /- run a garbage collection (if enabled) + if[gc;.gc.run[]]; + }; + +/- initialise reloadsummary, keyed table to track status of local reloads reloadsummary:([handle:`int$()]process:`symbol$();status:`boolean$();result:`symbol$()); doreload:{[pt] - .wdb.reloadcomplete:0b; - /-inform gateway of reload start - informgateway(`reloadstart;`); - getprocs[;pt] each reloadorder; - if[eodwaittime>0; - .timer.one[.wdb.timeouttime:.proc.cp[]+.wdb.eodwaittime;(value;".wdb.flushend[]");"release all hdbs and rdbs as timer has expired";0b]; - ]; - }; + reloadcomplete::0b; + /-inform gateway of reload start + informgateway(`reloadstart;`); + getprocs[;pt] each reloadorder; + if[eodwaittime>0; + .timer.one[timeouttime::.proc.cp[]+eodwaittime;(value;".wdb.flushend[]");"release all hdbs and rdbs as timer has expired";0b]; + ]; + }; // set .z.zd to control how data gets compressed setcompression:{[compression] if[3=count compression; - .lg.o[`compression;$[compression~16 0 0;"resetting";"setting"]," compression level to (",(";" sv string compression),")"]; - .dotz.set[`.z.zd;compression] - ]} + .lg.o[`compression;$[compression~16 0 0;"resetting";"setting"]," compression level to (",(";" sv string compression),")"]; + .dotz.set[`.z.zd;compression] + ]} resetcompression:{setcompression 16 0 0 } //check if the hdb directory contains current partition //if yes check if patition is empty and if it is not see if any of the tables exist in both the //temporary parition and the hdb partition. If there is a clash abort operation otherwise copy //each table to the hdb partition -movetohdb:{[dw;hw;pt] +movetohdb:{[dw;hw;pt] $[not(`$string pt)in key hsym`$-10 _ hw; .[.os.ren;(dw;hw);{.lg.e[`mvtohdb;"Failed to move data from wdb ",x," to hdb directory ",y," : ",z]}[dw;hw]]; not any a[dw]in(a:{key hsym`$x}) hw; - [{[y;x] + [{[y;x] $[not(b:`$last"/"vs x)in key y; [.[.os.ren;(x;y);{[x;y;e].lg.e[`mvtohdb;"Table ",string[x]," has failed to copy to ",string[y]," with error: ",e]}[b;y;]]; .lg.o[`mvtohdb;"Table ",string[b]," has been successfully moved to ",string[y]]]; @@ -273,7 +305,7 @@ endofdaysortdate:{[dir;pt;tablist;hdbsettings] reloadsymfile[.Q.dd[hdbsettings `hdbdir;`sym]]; {[x] .sort.sorttab[x];if[gc;.gc.run[]]} each tablist,'.Q.par[dir;pt;] each tablist]]; .lg.o[`sort;"finished sorting data"]; - + /-move data into hdb .lg.o[`mvtohdb;"Moving partition from the temp wdb ",(dw:.os.pth -1 _ string .Q.par[dir;pt;`])," directory to the hdb directory ",hw:.os.pth -1 _ string .Q.par[hdbsettings[`hdbdir];pt;`]]; .lg.o[`mvtohdb;"Attempting to move ",(", "sv string key hsym`$dw)," from ",dw," to ",hw]; @@ -281,17 +313,21 @@ endofdaysortdate:{[dir;pt;tablist;hdbsettings] /-call the posteod function .save.postreplay[hdbsettings[`hdbdir];pt]; - if[permitreload; + if[permitreload; doreload[pt]; ]; }; -merge:{[dir;pt;tableinfo;mergelimits;hdbsettings;mergemethod] +merge:{[dir;pt;tableinfo;mergelimits;hdbsettings;mergemethod;writedownmode] setcompression[hdbsettings[`compression]]; /- get tablename tabname:tableinfo[0]; - /- get list of partition directories for specified table - partdirs:` sv' tabledir,/:k:key tabledir:.Q.par[hsym dir;pt;tabname]; + /- get list of partition directories for specified table - partbyenum uses different folder structure vs partbyattr/default + partdirs:$[writedownmode in `partbyenum; + p where 0// +/- this function deletes
folder or the whole if it only has one element +removetablefromenumdir:{[partdir] + enumdir:` sv -1_` vs partdir; + .os.deldir .os.pth string $[1=count key enumdir;enumdir;partdir]; + }; + +endofdaymerge:{[dir;pt;tablist;mergelimits;hdbsettings;mergemethod;writedownmode] /- merge data from partitons /- .z.pd funciton in finspace will cause an error. Add in this check to skip over the use of .z.pd. This should be temporary and will be removed when issue resolved by AWS. tempfix2:$[.finspace.enabled;0b;(0 < count .z.pd[])]; $[tempfix2 and ((system "s")<0); [.lg.o[`merge;"merging on worker"]; {(neg x)(`.wdb.reloadsymfile;y);(neg x)(::)}[;.Q.dd[hdbsettings `hdbdir;`sym]] each .z.pd[]; - /-upsert .merge.partsize data to sort workers, only needed for part and hybrid method + /-upsert .merge.partsize data to sort workers, only needed for part and hybrid method if[(mergemode~`hybrid)or(mergemode~`part); {(neg x)(upsert;`.merge.partsizes;y);(neg x)(::)}[;.merge.partsizes] each .z.pd[]; ]; - merge[dir;pt;;mergelimits;hdbsettings;mergemethod] peach flip (key tablist;value tablist); + merge[dir;pt;;mergelimits;hdbsettings;mergemethod;writedownmode] peach flip (key tablist;value tablist); /-clear out in memory table, .merge.partsizes, and call sort worker processes to do the same .lg.o[`eod;"Delete from partsizes"]; delete from `.merge.partsizes; @@ -344,182 +391,202 @@ endofdaymerge:{[dir;pt;tablist;mergelimits;hdbsettings;mergemethod] delete from `.merge.partsizes; /- run a garbage collection if enabled if[gc;.gc.run[]]};`);(neg x)(::)} each .z.pd[]; - ]; + ]; [.lg.o[`merge;"merging on main"]; reloadsymfile[.Q.dd[hdbsettings `hdbdir;`sym]]; - merge[dir;pt;;mergelimits;hdbsettings;mergemethod] each flip (key tablist;value tablist); + merge[dir;pt;;mergelimits;hdbsettings;mergemethod;writedownmode] each flip (key tablist;value tablist); .lg.o[`eod;"Delete from partsizes"]; delete from `.merge.partsizes; ] ]; /- if path exists, delete it - if[not () ~ key savedir; + if[not () ~ key savedir; .lg.o[`merge;"deleting temp storage directory"]; .os.deldir .os.pth[string[` sv savedir,`$string[pt]]]; ]; /-call the posteod function .save.postreplay[hdbsettings[`hdbdir];pt]; - $[permitreload; + $[permitreload; doreload[pt]; if[gc;.gc.run[]]; ]; }; - + /- end of day sort [depends on writedown mode] endofdaysort:{[dir;pt;tablist;writedownmode;mergelimits;hdbsettings;mergemethod] - /- set compression level (.z.zd) - setcompression[hdbsettings[`compression]]; - $[writedownmode~`partbyattr; - endofdaymerge[dir;pt;tablist;mergelimits;hdbsettings;mergemethod]; - endofdaysortdate[dir;pt;key tablist;hdbsettings] - ]; - /- reset compression level (.z.zd) - resetcompression[16 0 0] - }; + /- set compression level (.z.zd) + setcompression[hdbsettings[`compression]]; + $[writedownmode in partwritemodes; + endofdaymerge[dir;pt;tablist;mergelimits;hdbsettings;mergemethod;writedownmode]; + endofdaysortdate[dir;pt;key tablist;hdbsettings] + ]; + /- reset compression level (.z.zd) + resetcompression[16 0 0] + }; /-function to send reload message to rdbs/hdbs reloadproc:{[h;d;ptype] - /-count of processes to be reloaded - .wdb.countreload:count[raze .servers.getservers[`proctype;;()!();1b;0b]each reloadorder]; - /-defining lambdas to be in asynchronously calling processes to reload + /-count of processes to be reloaded + countreload::count[raze .servers.getservers[`proctype;;()!();1b;0b]each reloadorder]; + /-defining lambdas to be in asynchronously calling processes to reload /-async call back function executed when eodwaittime>0 sendfunc:{[x;y;ptype].[{neg[y]@x};(x;y);{[ptype;x].lg.e[`reloadproc;"failed to reload the ",string[ptype]];'x}[ptype]]}; - /-reload function sent to processes by sendfunc in order to call process to reload. If process fail to reload log error + /-reload function sent to processes by sendfunc in order to call process to reload. If process fail to reload log error /-and call .wdb.handler with failed reload message. If reload is successful call .wdb.handler with successful reload message. reloadfunc:{[d;ptype] r:@[{(1b;`. `reload x)};d;{.lg.e[`reloadproc;"failed to reload from .wdb.reloadproc call. The error was : ",x];(0b;x)}]; (neg .z.w)(`.wdb.handler;(ptype;first r;$[first r;`$"reloaded successfully";`$"reload failed with error ",last r]));(neg .z.w)[]}; /-reload function to be executed if eodwaitime = 0 - sync message processes to reload and log if reload was successful or failed syncreloadfunc:{[h;d;ptype] r:@[h;({(1b;`reload x)};d);{[ptype;e] .lg.e[`reloadproc;"failed to reload the ",string[ptype],". The error was : ",e];(0b;e)}[ptype]]; - .lg.o[`reloadproc;"the ", string[ptype]," ", $[first r; "successfully reloaded"; "failed to reload with error ",last r]]}; + .lg.o[`reloadproc;"the ", string[ptype]," ", $[first r; "successfully reloaded"; "failed to reload with error ",last r]]}; .lg.o[`reloadproc;"sending reload call to ", string[ptype]]; $[eodwaittime>0; - sendfunc[(reloadfunc;d;ptype);h;ptype]; - syncreloadfunc[h;d;ptype] + sendfunc[(reloadfunc;d;ptype);h;ptype]; + syncreloadfunc[h;d;ptype] ]; } /-function to discover rdbs/hdbs and attempt to reconnect getprocs:{[x;y] - a:exec (w!x) from .servers.getservers[`proctype;x;()!();1b;0b]; - /-exit if no valid handle - if[0=count a; .lg.e[`connection;"no connection to the ",(string x)," could be established... failed to reload ",string x];:()]; - .lg.o[`connection;"connection to the ", (string x)," has been located"]; - /-send message along each handle a - reloadproc[;y;value a] each key a; - } + a:exec (w!x) from .servers.getservers[`proctype;x;()!();1b;0b]; + /-exit if no valid handle + if[0=count a; .lg.e[`connection;"no connection to the ",(string x)," could be established... failed to reload ",string x];:()]; + .lg.o[`connection;"connection to the ", (string x)," has been located"]; + /-send message along each handle a + reloadproc[;y;value a] each key a; + } /-function to send messages to gateway informgateway:{[message] - .lg.o[`informgateway;"sending message to gateway(s)"]; - $[count gateways:.servers.getservers[`proctype;gatewaytypes;()!();1b;0b]; - [ - {.[@;(y;x);{.lg.e[`informgateway;"unable to run command on gateway"];'x}]}[message;] each exec w from gateways; - .lg.o[`informgateway;"the message - ",(.Q.s message), " was sent to the gateways"] - ]; - .lg.e[`informgateway;"can't connect to the gateway - no gateway detected"]] - } - + .lg.o[`informgateway;"sending message to gateway(s)"]; + $[count gateways:.servers.getservers[`proctype;gatewaytypes;()!();1b;0b]; + [ + {.[@;(y;x);{.lg.e[`informgateway;"unable to run command on gateway"];'x}]}[message;] each exec w from gateways; + .lg.o[`informgateway;"the message - ",(.Q.s message), " was sent to the gateways"] + ]; + .lg.e[`informgateway;"can't connect to the gateway - no gateway detected"]] + } + /- function to call that will cause sort & reload process to sort data and reload rdb and hdbs informsortandreload:{[dir;pt;tablist;writedownmode;mergelimits;hdbsettings;mergemethod] .lg.o[`informsortandreload;"attempting to contact sort process to initiate data ",$[writedownmode~`default;"sort";"merge"]]; - $[count sortprocs:.servers.getservers[`proctype;sorttypes;()!();1b;0b]; - [if[(mergemode~`hybrid)or(mergemode~`part); - // for part and hybrid method sort procs need access to partsizes table data - upsert data tp sort procs - {(neg x)(upsert;`.merge.partsizes;y);(neg x)(::)}[;.merge.partsizes] each exec w from sortprocs; - ]; - {.[{neg[y]@x;neg[y][]};(x;y);{.lg.e[`informsortandreload;"unable to run command on sort and reload process"];'x}]}[(`.wdb.endofdaysort;dir;pt;tablist;writedownmode;mergelimits;hdbsettings;mergemethod);] each exec w from sortprocs; - ]; - [.lg.e[`informsortandreload;"can't connect to the sortandreload - no sortandreload process detected"]; - // try to run the sort locally - endofdaysort[dir;pt;tablist;writedownmode;mergelimits;hdbsettings;mergemethod]]]; - }; + $[count sortprocs:.servers.getservers[`proctype;sorttypes;()!();1b;0b]; + [if[(mergemode~`hybrid)or(mergemode~`part); + // for part and hybrid method sort procs need access to partsizes table data - upsert data tp sort procs + {(neg x)(upsert;`.merge.partsizes;y);(neg x)(::)}[;.merge.partsizes] each exec w from sortprocs; + ]; + {.[{neg[y]@x;neg[y][]};(x;y);{.lg.e[`informsortandreload;"unable to run command on sort and reload process"];'x}]}[(`.wdb.endofdaysort;dir;pt;tablist;writedownmode;mergelimits;hdbsettings;mergemethod);] each exec w from sortprocs; + ]; + [.lg.e[`informsortandreload;"can't connect to the sortandreload - no sortandreload process detected"]; + // try to run the sort locally + endofdaysort[dir;pt;tablist;writedownmode;mergelimits;hdbsettings;mergemethod]]]; + }; /-function to set the timer for the save to disk function starttimer:{[] - $[@[value;`.timer.enabled;0b]; - [.lg.o[`init;"adding the wdb save to disk function to the timer"]; - /-add .wdb.savetodisk function to TorQ timer - .timer.repeat[.proc.cp[];0Wp;settimer;(`.wdb.savetodisk;`);"save wdb data to disk"]; - .lg.o[`init;"the timer has been set to ", string settimer]]; - /-if timer not enabled, prompt user to enable it - .lg.e[`init;"the timer has not been enabled - please enable the timer to run the wdb"]]; - } - -/-function to subscribe to tickerplant + $[@[value;`.timer.enabled;0b]; + [.lg.o[`init;"adding the wdb save to disk function to the timer"]; + /-add .wdb.savetodisk function to TorQ timer + .timer.repeat[.proc.cp[];0Wp;settimer;(`.wdb.savetodisk;`);"save wdb data to disk"]; + .lg.o[`init;"the timer has been set to ", string settimer]]; + /-if timer not enabled, prompt user to enable it + .lg.e[`init;"the timer has not been enabled - please enable the timer to run the wdb"]]; + } + +/-function to subscribe to tickerplant subscribe:{[] - s:.sub.getsubscriptionhandles[tickerplanttypes;();()!()]; - if[count s; - .lg.o[`subscribe;"tickerplant found - subscribing to ", string (subproc: first s)`procname]; - /- return the tables subscribed to and the tickerplant log date - subto:.sub.subscribe[subtabs;subsyms;schema;replay;subproc]; - /- check the tp logdate against the current date and correct if necessary - fixpartition[subto];];} - + s:.sub.getsubscriptionhandles[tickerplanttypes;();()!()]; + if[count s; + .lg.o[`subscribe;"tickerplant found - subscribing to ", string (subproc: first s)`procname]; + /- return the tables subscribed to and the tickerplant log date + subto:.sub.subscribe[subtabs;subsyms;schema;replay;subproc]; + /- check the tp logdate against the current date and correct if necessary + fixpartition[subto];]; + } + /- function to rectify data written to wrong partition -fixpartition:{[subto] - /- check if the tp logdate matches current date - if[not (tplogdate:subto[`tplogdate])~orig:.wdb.currentpartition; - .lg.o[`fixpartition;"Current partiton date does not match the ticker plant log date"]; - /- set the current partiton date to the log date - .wdb.currentpartition:tplogdate; - /- move the data that has been written to correct partition - pth1:.os.pth[-1 _ string .Q.par[savedir;orig;`]]; - pth2:.os.pth[-1 _ string .Q.par[savedir;tplogdate;`]]; - if[not ()~key hsym `$.os.pthq pth1; - /- delete any data in the current partiton directory - clearwdbdata[]; - .lg.o[`fixpartition;"Moving data from partition ",(.os.pthq pth1) ," to partition ",.os.pthq pth2]; - .[.os.ren;(pth1;pth2);{.lg.e[`fixpartition;"Failed to move data from wdb partition ",x," to wdb partition ",y," : ",z]}[pth1;pth2]]]; - ]; - } +fixpartition:{[subto] + /- check if the tp logdate matches current date + if[not (tplogdate:subto[`tplogdate])~orig:currentpartition; + .lg.o[`fixpartition;"Current partiton date does not match the ticker plant log date"]; + /- set the current partiton date to the log date + currentpartition::tplogdate; + /- move the data that has been written to correct partition + pth1:.os.pth[-1 _ string .Q.par[savedir;orig;`]]; + pth2:.os.pth[-1 _ string .Q.par[savedir;tplogdate;`]]; + if[not ()~key hsym `$.os.pthq pth1; + /- delete any data in the current partiton directory + clearwdbdata[]; + .lg.o[`fixpartition;"Moving data from partition ",(.os.pthq pth1) ," to partition ",.os.pthq pth2]; + .[.os.ren;(pth1;pth2);{.lg.e[`fixpartition;"Failed to move data from wdb partition ",x," to wdb partition ",y," : ",z]}[pth1;pth2]]]; + ]; + } + +/- for writedown modes partbyenum/default we make sure that partition 0/currentpartition has all the tables. +/- In that case we can use .Q.chk later to fill the db making it useable for intraday processes +initmissingtables:{[] + .lg.o[`fixpartition;"Adding missing tables(empty) to partition ",string currentpartition]; + inittable each tablelist[]; + filldb[]; + } + +filldb:{[] + /- for all enumerated partitions we want to make sure that all tables are present + .Q.chk[.Q.par[hsym savedir; currentpartition; `]]; + } + +/- initialises table t in db with its schema in part +inittable:{[t] + tabledir:` sv $[writedownmode~`partbyenum; .Q.par[.Q.dd[hsym savedir;currentpartition];0;t]; .Q.par[hsym savedir;currentpartition;t]],`; + if[() ~ key tabledir;tabledir set .Q.en[hsym hdbdir;0#value t]]; + } /- will check on each upd to determine where data should be flushed to disk (if max row limit has been exceeded) replayupd:{[f;t;d] - /- execute the supplied function + /- execute the supplied function f . (t;d); - /- if the data count is greater than the threshold, then flush data to disk - if[(rpc:count[value t]) > lmt:maxrows[t]; - .lg.o[`replayupd;"row limit (",string[lmt],") exceeded for ",string[t],". Table count is : ",string[rpc],". Flushing table to disk..."]; - savetables[savedir;getpartition[];0b;t]] - }[upd]; + /- if the data count is greater than the threshold, then flush data to disk + if[(rpc:count[value t]) > lmt:maxrows[t]; + .lg.o[`replayupd;"row limit (",string[lmt],") exceeded for ",string[t],". Table count is : ",string[rpc],". Flushing table to disk..."]; + savetables[savedir;getpartition[];0b;t]] + }[upd]; / - if there is data in the wdb directory for the partition, if there is remove it before replay / - is only for wdb processes that are saving data to disk -clearwdbdata:{[] - $[saveenabled and not () ~ key wdbpart:.Q.par[savedir;getpartition[];`]; - [.lg.o[`deletewdbdata;"removing wdb data (",(delstrg:1_string wdbpart),") prior to log replay"]; - @[.os.deldir;delstrg;{[e] .lg.e[`deletewdbdata;"Failed to delete existing wdb data. Error was : ",e];'e }]; - .lg.o[`deletewdbdata;"finished removing wdb data prior to log replay"]; - ]; - .lg.o[`deletewdbdata;"no directory found at ",1_string wdbpart] - ]; - }; - +clearwdbdata:{[] + $[saveenabled and not () ~ key wdbpart:.Q.par[savedir;getpartition[];`]; + [.lg.o[`deletewdbdata;"removing wdb data (",(delstrg:1_string wdbpart),") prior to log replay"]; + @[.os.deldir;delstrg;{[e] .lg.e[`deletewdbdata;"Failed to delete existing wdb data. Error was : ",e];'e }]; + .lg.o[`deletewdbdata;"finished removing wdb data prior to log replay"]; + ]; + .lg.o[`deletewdbdata;"no directory found at ",1_string wdbpart] + ]; + }; + / - function to check that the tickerplant is connected and subscription has been setup notpconnected:{[] - 0 = count select from .sub.SUBSCRIPTIONS where proctype in .wdb.tickerplanttypes, active} + 0 = count select from .sub.SUBSCRIPTIONS where proctype in tickerplanttypes, active} getsortparams:{[] - /- get the attributes csv file - /-even if running with a sort process should read this file to cope with backups - .sort.getsortcsv[.wdb.sortcsv]; - /- check the sort.csv for parted attributes `p if the writedownmode `partbyattr is selected - /- if each table does not have at least one `p attribute the process will exit - if[writedownmode~`partbyattr; - - /- check that default table is defined - if[not count exec distinct tabname from .sort.params where tabname=`default,att=`p,sort=1b; - .lg.e[`init;"default table not defined in sort.csv with at least one `p attribute and sort=1b"]; - ]; - .lg.o[`init;"default table defined in sort.csv and with at least one `p attribute and sort=1b"]; - - /- check for `p attributes - if[count notparted:distinct .sort.params[`tabname] except distinct exec tabname from .sort.params where att in `p; - .lg.e[`init;"parted attribute p not set at least once in sort.csv for table(s): ", ", " sv string notparted]; - ]; - .lg.o[`init;"parted attribute p set at least once for each table in sort.csv"]; - ]; - }; + /- get the attributes csv file + /-even if running with a sort process should read this file to cope with backups + .sort.getsortcsv[sortcsv]; + /- check the sort.csv for parted attributes `p if the writedownmode `partbyattr or `partbyenum is selected + /- if each table does not have at least one `p attribute the process will exit + if[writedownmode in partwritemodes; + + /- check that default table is defined + if[not count exec distinct tabname from .sort.params where tabname=`default,att=`p,sort=1b; + .lg.e[`init;"default table not defined in sort.csv with at least one `p attribute and sort=1b"]; + ]; + .lg.o[`init;"default table defined in sort.csv and with at least one `p attribute and sort=1b"]; + + /- check for `p attributes + if[count notparted:distinct .sort.params[`tabname] except distinct exec tabname from .sort.params where att in `p; + .lg.e[`init;"parted attribute p not set at least once in sort.csv for table(s): ", ", " sv string notparted]; + ]; + .lg.o[`init;"parted attribute p set at least once for each table in sort.csv"]; + ]; + }; \d . @@ -528,15 +595,15 @@ getsortparams:{[] /- make sure to request connections for all the correct types -.servers.CONNECTIONS:(distinct .servers.CONNECTIONS,.wdb.hdbtypes,.wdb.rdbtypes,.wdb.gatewaytypes,.wdb.tickerplanttypes,.wdb.sorttypes,.wdb.sortworkertypes) except ` +.servers.CONNECTIONS:(distinct .servers.CONNECTIONS,.wdb.hdbtypes,.wdb.rdbtypes,.wdb.gatewaytypes,.wdb.tickerplanttypes,.wdb.sorttypes,.wdb.sortworkertypes,.wdb.idbtypes) except `; /- adds endofday function to top level namespace endofday: .wdb.endofday; /- setting the upd and .u.end functions as the .wdb versions .u.end:{[pt] - .wdb.endofday[.wdb.getpartition[];()!()]; + .wdb.endofday[.wdb.getpartition[];()!()]; } - + /- set the replay upd .lg.o[`init;"setting the log replay upd function"]; upd:.wdb.replayupd; diff --git a/code/wdb/origstartup.q b/code/wdb/origstartup.q index 3f7cd4d50..db884fa29 100644 --- a/code/wdb/origstartup.q +++ b/code/wdb/origstartup.q @@ -1,17 +1,21 @@ \d .wdb startup:{[] - .lg.o[`init;"searching for servers"]; - .servers.startup[]; - if[writedownmode~`partbyattr; - .lg.o[`init;"writedown mode set to ",(string .wdb.writedownmode)] - ]; - .lg.o[`init;"partition has been set to [savedir]/[", (string partitiontype),"]/[tablename]/", $[writedownmode~`partbyattr;"[parted column(s)]/";""]]; - if[saveenabled; - //check if tickerplant is available and if not exit with error - if[not .finspace.enabled; /-TODO Remove when tickerplant fixed in finspace - .servers.startupdepcycles[.wdb.tickerplanttypes;.wdb.tpconnsleepintv;.wdb.tpcheckcycles]; - ]; - subscribe[]; - ]; - @[`.;`upd;:;.wdb.upd]; - } + .lg.o[`init; "searching for servers"]; + .servers.startup[]; + .lg.o[`init; "writedown mode set to ",(string .wdb.writedownmode)]; + $[writedownmode~`partbyattr; + .lg.o[`init; "partition has been set to [savedir]/[",(string partitiontype),"]/[tablename]/[parted column(s)]/"]; + writedownmode~`partbyenum; + .lg.o[`init; "partition has been set to [savedir]/[",(string partitiontype),"]/[parted symbol column enumerated]/[tablename]/"]; + .lg.o[`init; "partition has been set to [savedir]/[",(string partitiontype),"]/[tablename]/"]]; + if[saveenabled; + //check if tickerplant is available and if not exit with error + if[not .finspace.enabled; /-TODO Remove when tickerplant fixed in finspace + .servers.startupdepcycles[.wdb.tickerplanttypes; .wdb.tpconnsleepintv; .wdb.tpcheckcycles]; + ]; + subscribe[]; + /- add missing tables to partitions in case an IDB process wants to connect. Only applicable for partbyenum writedown mode + if[.wdb.writedownmode in `default`partbyenum;initmissingtables[]]; + ]; + @[`.; `upd; :; .wdb.upd]; + } \ No newline at end of file diff --git a/code/wdb/writedown.q b/code/wdb/writedown.q index c27bd6868..d0ee2566d 100644 --- a/code/wdb/writedown.q +++ b/code/wdb/writedown.q @@ -2,13 +2,12 @@ /-Required variables for savetables function compression:@[value;`compression;()]; /-specify the compress level, empty list if no required -savedir:@[value;`savedir;`:temphdb]; /-location to save wdb data -hdbdir:@[value;`hdbdir;`:hdb]; /-move wdb database to different location +savedir:hsym @[value;`savedir;`:temphdb]; /-location to save wdb data +hdbdir:hsym @[value;`hdbdir;`:hdb]; /-move wdb database to different location -hdbsettings:(`compression`hdbdir)!(compression;hdbdir); +hdbsettings:(`compression`hdbdir)!(compression;hsym hdbdir); numrows:@[value;`numrows;100000]; /-default number of rows numtab:@[value;`numtab;`quote`trade!10000 50000]; /-specify number of rows per table -gmttime:@[value;`gmttime;1b]; /-define whether the process is on gmttime or not maxrows:{[tabname] numrows^numtab[tabname]}; /- extract user defined row counts @@ -18,7 +17,7 @@ partitiontype:@[value;`partitiontype;`date]; /-set getpartition:@[value;`getpartition; /-function to determine the partition value {{@[value;`.wdb.currentpartition; - (`date^partitiontype)$(.z.D,.z.d)gmttime]}}]; + (`date^partitiontype)$.proc.cd[]]}}]; currentpartition:.wdb.getpartition[]; /- Initialise current partiton @@ -38,13 +37,14 @@ savetables:{[dir;pt;forcesave;tabname] ]; /- make addition to tabsizes .lg.o[`track;"appending table details to tabsizes"]; - .wdb.tabsizes+:([tablename:enlist tabname]rowcount:enlist arows;bytes:enlist -22!r); + .wdb.tabsizes+:([tablename:enlist tabname]rowcount:enlist arows;bytes:enlist -22!r); /- empty the table .lg.o[`delete;"deleting ",(string tabname)," data from in-memory table"]; @[`.;tabname;0#]; /- run a garbage collection (if enabled) if[gc;.gc.run[]]; - ]}; + :1b; + ]; 0b}; \d . /-endofperiod function diff --git a/config/passwords/idb.txt b/config/passwords/idb.txt new file mode 100644 index 000000000..272ed7758 --- /dev/null +++ b/config/passwords/idb.txt @@ -0,0 +1 @@ +idb:pass diff --git a/config/settings/gateway.q b/config/settings/gateway.q index 820c6c9bc..8787e7e4c 100644 --- a/config/settings/gateway.q +++ b/config/settings/gateway.q @@ -4,21 +4,21 @@ // if error & sync message, throws an error. Else passes result as normal // status - 1b=success, 0b=error. sync - 1b=sync, 0b=async formatresponse:{[status;sync;result]$[not[status]and sync;'result;result]}; -synccallsallowed:0b // whether synchronous calls are allowed -querykeeptime:0D00:30 // the time to keep queries in the -errorprefix:"error: " // the prefix for clients to look for in error strings -clearinactivetime:0D01:00 // the time to keep inactive handle data +synccallsallowed:0b // whether synchronous calls are allowed +querykeeptime:0D00:30 // the time to keep queries in the +errorprefix:"error: " // the prefix for clients to look for in error strings +clearinactivetime:0D01:00 // the time to keep inactive handle data \d .kxdash -enabled:0b // Functionality for parsing and handling kx dashboard queries - disabled by default +enabled:0b // Functionality for parsing and handling kx dashboard queries - disabled by default \d .proc loadprocesscode:1b // whether to load the process specific code defined at ${KDBCODE}/{process type} // Server connection details \d .servers -CONNECTIONS:`rdb`hdb // list of connections to make at start up +CONNECTIONS:`rdb`hdb`idb // list of connections to make at start up RETRY:0D00:01 // period on which to retry dead connections. If 0, no reconnection attempts \d .aqrest -loadexecute:0b // Whether to reset .aqrest.execute +loadexecute:0b // Whether to reset .aqrest.execute diff --git a/config/settings/idb.q b/config/settings/idb.q new file mode 100644 index 000000000..f37d53b62 --- /dev/null +++ b/config/settings/idb.q @@ -0,0 +1,11 @@ +// Bespoke IDB config +\d .idb +wdbtypes:`wdb; + +// Server connection details +\d .servers +CONNECTIONS:`wdb // list of connections to make at start up +STARTUP:1b // create connections + +\d .proc +loadprocesscode:0b // whether to load the process specific code defined at ${KDBCODE}/{process type} diff --git a/config/settings/sort.q b/config/settings/sort.q index b32c3d7f7..8f3233700 100644 --- a/config/settings/sort.q +++ b/config/settings/sort.q @@ -23,9 +23,9 @@ mode:`sort // the wdb process can operate in three // save mode process. When this is triggered it will sort the // data on disk, apply attributes and the trigger a reload on the // rdb and hdb processes - -mergenumrows:100000 // default number of rows for merge process -mergenumtab:`quote`trade!10000 50000 // specify number of rows per table + +mergenumrows:100000 // default number of rows for merge process +mergenumtab:`quote`trade!10000 50000 // specify number of rows per table tpconnsleepintv:10 // number of seconds between attempts to connect to the tp upd:insert // value of the upd function @@ -33,19 +33,18 @@ replay:1b // replay the tickerplant log file schema:1b // retrieve schema from tickerplant settimer:0D00:00:10 // timer to check if data needs written to disk partitiontype:`date // set type of partition (defaults to `date, can be `date, `month or `year) -gmttime:1b // define whether the process is on gmttime or not getpartition:{@[value; - `.wdb.currentpartition; - (`date^partitiontype)$(.z.D,.z.d)gmttime]} //function to determine the partition value + `.wdb.currentpartition; + (`date^partitiontype)$.proc.cd[]]} //function to determine the partition value reloadorder:`hdb`rdb // order to reload hdbs and rdbs hdbdir:`:hdb // move wdb database to different location sortcsv:hsym first .proc.getconfigfile["sort.csv"] // location of csv file permitreload:1b // enable reload of hdbs/rdbs compression:() // specify the compress level, empty list if no required gc:1b // garbage collect at appropriate points (after each table save and after sorting data) -eodwaittime:0D00:00:10.000 // time to wait for async calls to complete at eod +eodwaittime:0D00:00:10.000 // time to wait for async calls to complete at eod // Server connection details \d .servers -CONNECTIONS:`hdb`tickerplant`rdb`gateway // list of connections to make at start up -STARTUP:1b // create connections +CONNECTIONS:`hdb`tickerplant`rdb`gateway // list of connections to make at start up +STARTUP:1b // create connections diff --git a/config/settings/wdb.q b/config/settings/wdb.q index 1c9d12390..16250689d 100644 --- a/config/settings/wdb.q +++ b/config/settings/wdb.q @@ -5,6 +5,7 @@ ignorelist:`heartbeat`logmsg // list of tables to ignore hdbtypes:`hdb // list of hdb types to look for and call in hdb reload rdbtypes:`rdb // list of rdb types to look for and call in rdb reload +idbtypes:`idb // list of idb types to look for and call in rdb reload gatewaytypes:`gateway // list of gateway types to inform at reload tickerplanttypes:`segmentedtickerplant // list of tickerplant types to try and make a connection to subtabs:` // list of tables to subscribe for (` for all) @@ -30,6 +31,10 @@ writedownmode:`default // at EOD the data will be sorted and given attributes according to sort.csv before being moved to hdb // 2. partbyattr - the data is partitioned by [ partitiontype ] and the column(s)assigned the parted attributed in sort.csv // at EOD the data will be merged from each partiton before being moved to hdb + // 3. partbyenum - the data is partitioned by [ partitiontype ] and a symbol column with parted attrobution assigned in sort.csv + // at EOD the data will be merged from each partiton before being moved to hdb +enumcol:`sym; // default column for partitioning. Only used with writedownmode: partbyenum. + mergemode:`part // the partbyattr writdown mode can merge data from tenmporary storage to the hdb in three ways: // 1. part - the entire partition is merged to the hdb // 2. col - each column in the temporary partitions are merged individually @@ -45,8 +50,7 @@ replay:1b schema:1b // retrieve schema from tickerplant settimer:0D00:00:10 // timer to check if data needs written to disk partitiontype:`date // set type of partition (defaults to `date, can be `date, `month or `year) -gmttime:1b // define whether the process is on gmttime or not -getpartition:{@[value;`.wdb.currentpartition;(`date^partitiontype)$(.z.D,.z.d)gmttime]} // function to determine the partition value +getpartition:{@[value;`.wdb.currentpartition;(`date^partitiontype)$.proc.cd[]]} // function to determine the partition value reloadorder:`hdb`rdb // order to reload hdbs and rdbs hdbdir:`:hdb // move wdb database to different location sortcsv:hsym first .proc.getconfigfile"sort.csv" // location of csv file @@ -58,9 +62,9 @@ tpcheckcycles:0W // Server connection details \d .servers -CONNECTIONS:`hdb`tickerplant`rdb`gateway`sort // list of connections to make at start up STARTUP:1b // create connections +CONNECTIONS:`hdb`tickerplant`rdb`gateway`sort // list of connections to make at start up \d .proc -loadprocesscode:1b // Whether to load the process specific code defined at ${KDBCODE}/{process type} +loadprocesscode:1b // Whether to load the process specific code defined at ${KDBCODE}/{process type} diff --git a/docs/Processes.md b/docs/Processes.md index 4abdb1725..4ebf3f335 100755 --- a/docs/Processes.md +++ b/docs/Processes.md @@ -988,7 +988,7 @@ process should be configured in the processes.csv file with a proctype of sort. The save process will check for processes with a proctype of sort when it attempts to trigger the end of day sort of the data. -The wdb process provides two methods for persisting data to disk and +The wdb process provides three methods for persisting data to disk and sorting at the end of the day. - default - Data is persisted into a partition defined by the @@ -1019,14 +1019,53 @@ sorting at the end of the day. need for sorting. The number of rows that are joined at once is limited by the mergenumrows and mergenumtab parameters. -The optional partbyattr method may provide a significant saving in time -at the end of day, allowing the hdb to be accessed sooner. For large +- partbyenum - Data is persisted to a partition scheme where the partition + is derived from parameters in the sort.csv file. In this mode partition + only can be done by one column which has parted attribute applied on it + and it also has to be of a symbol type. The partitioning on disk will + be the enumerated symbol entries of the parted symbol column. The + enumeration is done against the HDB sym file. + The general partition scheme is of the form + \[wdbdir\]/\[partitiontype\]/\[parted enumerated symbol column\]/\[table(s)\]/. + A typical partition directory would be similar to(for ex sym: MSFT_N) + wdb/database/2015.11.26/456/trade/ + In the above example, the data is parted by sym, and number 456 is + the order of MSFT_N symbol entry in the HDB sym file. + + The advantage of partbyenum over partbyattr could be that the + directory structure it uses represents a HDB that is ready to be loaded + intraday. At the end of the day the data gets upserted to the HDB the + same way it would be when using partbyattr. + +The optional partbyattr/partbyenum methods may provide a significant saving in +time at the end of day, allowing the hdb to be accessed sooner. For large data sets with a low cardinality (ie. small number of distinct elements) the optional method may provide a significant time saving, upwards of 50%. The optional method should also reduce the memory usage at the end of day event, as joining data is generally less memory intensive than sorting. + + +Intraday Database (IDB) +-------------------- + +The Intraday Database or IDB is a simple process that allows access to +data written down intraday. This assumes that there is an existing WDB +process creating a DB on disk that can be loaded with a simple load command. +As of now default and partbyenum WDB writedown modes are supported. +The responsibility of an IDB is therefore: + +1. Serving queries. Since partbyenum writedown mode is done by enumerated + symbol columns a helper function maptoint is implemented to support + symbol lookup in sym file: + select from trade where int=maptoint[`MSFT_N] + +2. Can be triggered for a reload. This is usually done by the WDB process + periodically. + +![IDB diagram](graphics/idb.png) + Tickerplant Log Replay diff --git a/docs/graphics/idb.png b/docs/graphics/idb.png new file mode 100644 index 0000000000000000000000000000000000000000..14ff0b409ba9678a489f3f741b3ae97b1756bacd GIT binary patch literal 40780 zcmeFZXIPWj_b!a%D5IzhDqsT~5J3Y(1VOrvG^rv;CsB%kQlysv(Q!~Z(vcPoO{GM7 z2}vvzseu4S2m}Q}ClLqpS{7@GzxhV)xGDq0;|jAMe+| zr1FdWn&)3q3nE4jF1|e{9_l$#vN8Myma>|K-wscu^S5tXAOBmSfx(E?%ZRqnuIR%1 z5lV@StuezLE9G5pv3u;v*QhXhq{1sOMGFYRI8 z_@($%pTG4mA{yJ)unYJq-Mqz4%)`XCvf#pI5A}mA$Q7A+?)?3|TD`tCP&V7bjoo*4 z25**cc{%LS@vu=Bi9%n?Jjbo*TXRlm35P9Bk8_d9f?80rD}uUu!7$V|`;qdAqrkYw zik=v>*g1?-jED_^?vv$e6IFK4>)(rHz(j8tP)6IGq)c;k*3u9BInc_a0gLMSQ#oOz z2SK~2C+X0vs0teU^p5-Loeu}GNdQs8eg#InU~?co5DWs>UX%e*6M9k&&*(Y&Z4SwW z=4e?lW_uCxu?aoW$FVJ6-_J+-$Y`0jHp4jaNOkA|F<)5nr*)FVjYzor{DC%lnN}to z7;o^UX4yoq7&7jn^vMWuA&H||;tgj&E3V^rZ5vnAD1ior2O`&Fzw+jxL`-yb{nX!2 z#2+cAU(#C#C*Vj@QI%Y!omq{Wld@pSk@BE5+MidDXRRycpMb5@bA)<@)4mm5hP$;Hxw_xOa-CfJSNBgr zw`ET*iVH>Se-j5?-`O)&!-PPvA;nl3o=$gmQX_UCNRkQ+99#<|>Dua>G#(FKN{ku9%NfpA z1ueAS^!}Z4)#~+W>DA}QbYO!9>ZRtL2#psaG0Jbi3A&&S;Ja7cRO=@apeS)Dwx~C6=T)OB{!=b=5U+5kLF{`oLWCD1_0QaeD zAIB}BDrh$T080OnRB@o)xc{HuRLS-Hh3KC_bcb2cLU=k9N#ISJ@^AAKPQjy9gt;RFmvrp( z_a7M!aH}aqjs|7HfcI|{KFNVOAZMMT>G6x=>bf>tLfE(isP>~1fLR^vmprVfJ9jYz zxVqwY24Zu|K*zN51$yA+kt!tUV&CCmy830;KvY&hokGiig$}Q3C&)fr0&aj7Jr!SW zE@x?qm}ESY#J#nfTh>ZGXFe$b#DLOskp`s?I%ke<8jrhA*ffXzo#)w3hXxCPLUnY zFX0+UX{E2X3}`fk#Htz>_IQz=rBgwOTWYicPwgLT4fkN8LUSXH9PiLe>POgU#+CsV z*S>(w9cTiax$YO#r_*u+g~utdqY-Q56@MFut`*g&TS_J}k#R7MyZVpWvNK_`_ zba-uAoR^!I!|#Z>${Kls4zXNJ9#wO5r0o&bwX-*jGv176+SpEac(qX)zo5QjzStrA z5{Dv|iyj1c)BT2YUtDCN?0y>A5kkE+_wZzqLh_PB#LDy1pv{BsC;Y9D^u&wOCh?c& z?J|!Va^vY{X0TMIVdl&lBtB^7mdar;BGzuKmR_{|KKESd2i*h?lDYL)I?nQ^19%Sh zVvt;)@fq*@4L?QE80dJ2v*X>~-LRtqqh}KmV}A!}c`;E`kt;E^H$A1bwppDg(!jo2Z;Ui@3sv6Av%K3`_BTpufaC<~ZH}@9 z0zfL_j{DH9rt{ngb*(YRsbWXBlhBPXuO~^{x3j~9On9&xKkwob^A`1INm$Br>P7d& zI=>akJ-OgtmUZ~h3Tubcj4b5zn~Zd+?a$D?72y@KWAbQYHG)#@T1f@EY5209$57+M z^=>%+r>~|2q6XSEx@(cLPMhkc3VYMDRMH&ne#ss%Z$$d&vV6}Nrr#yu4_VXXCpw=* zuA_1Se*}}_iIcPXFEg^V(ztQ&qj+J68CaHnnxT@s^oDrmtElAS5=HiN_xXxmIYYSE zc0nm^77G7qcB2;cc54US%`D{ln~a+_^`Zu>&oMtkz-lH;`r@nqBcESYC{tO#Wf> zd*u@uyJUMBF-jP;QR6m6bTW2g@6*EZWwtS6lNAWYRxdOLNR9a-zmoW^IC}V@i>k#x z;@kK+X33oXA30e8LU|VXVCM94g5Qy`#g%2Wf754P*HN)bY5%p=y>b@_jj{k0o9n5MgwB+ zV8Xv=oj05m2!H%ug=K&{CK z@c-NCo}v_~{*&N#E|?N%c|zph_u-OJi(GQte=b%$kzuI+pW&^j*JY6ZF|RzcMC9LV z`#;r*|Npfv0FP_~IV~2~mKaAD{plCgFDc|EhF;6}oG_6zBzGaR7tNG1^?&=*Dc_6$ z5No~ErK@zeUwG|z@d%ML1yEAah})F%Jp$``!_S;%+|tAi6{K&;M%P?w1Z_|2u7|;u zxTO&NC!WYHsP9CJcXvqWQVl-N3Wn#p+5Q*6Ba(9qLv=y}Q=y;h+KsUxmd#&STl!DD zk%Odg!sz0C7*AWVttT795LsS<1?RDVssyud2Z1F&$%H*O^=~aE%44IldEIzIj~m{M zR1kLm&c4&q*JQMyf&Pv_qFLO~l7uvai`<%A;5srKdfPe5{oD zDZ?^{c?cNdii~Q*L~lM&Y%TUx_!*N@^X>Ce>}&*%dc58YQ%v5Q@PTGj?(AoRGEO{P zyrx#DT}3A+26-B?AaqGZ6NP6Y#bL!0sp(R{P+lq7S+J}hsHzDc@3{#p!eS3s_;%Kl z?X%Q@3dKX;>iQqj9RT?8kXf6KK1zFi&!kPk>Wfi;ltUoQ@BQF}R+Vyp=dDy+0`~h|~SRd4ENX9fY8aujgd*I z(*9hHoNRJkjiVYc+=a{IZze#2G`Hd5(Wa^VL-~Ps#<_2gET7+A?>6KaAn9>F+KWrn z!{0rJCBX`&`g^qklzM!$`pfzaU6*{SWM~P8Wg%u&$xKL6EoS?DR}aku81;oqeRK;k z=U>n%Q`m}q>CIM!S*EOm$g#xV+&y|)<(zjAPtyKO$Di<~z!$1a(I!!Mykvh!p zkT!qU$f&kdq_KLH7%-@&QCBuh=x_)0%>KkEmR-NA;i@E2;wn}cB70{DfyCaJzJtz?$PGD|;`u`o2g2fgExbZp!nKVaC01M2R* zm_6|ue|zlty3ZP=ASzXSws6Yj$xdG&Hn#h=tX}eEgwNG9?9Kmyan2_;^&G`?R-K>O zn+Nu=0=pQ1$}`WCUl>DQDbLIq^j178wYHEuC7{h3m(xSeBq?fVg9uFU-3r#ofk|Fy&a2mTPZdjR{! z=IdylxmHIjU8I~nAI7+d~Zg^=5AjmFKl6Z)!7s)Zj5NmV2+2n~f&Klow=Rv70g8 zFSGKjr*Gw3R^fui*>M@ev%M%wIEz!~P0shvMksFr6S^__uL<2_j-RX>ZU`eI4)I1& z!@{9jsu*}_zff-jk~?J~Lnw2@W*pic6(7E1jlI#cXcwdD0%txY9Arkl#d8^Rsbw+) zTTx8nN;rMpy8&ZsVp@$2_%?Htl9-=trj@Lx=tAd^2zqz7L73Mo70-dGzX|NhaC z)Ye`+{JC!ZTw^T7K^L^zXfjU8<(tn8@Myu4E>b4&=(*^fS<_l`Ud5is7l+1#wX(3X z!(cGSOmm?gTsq2xoKtM+{L2&f@qUM)?A;jMZBFaHOhT0c!Gs6y}yESbquEDa%lA`y=8$MwTol;Lf-!iPA*8=@(j>g4A!K_9yaj(Q+nBdEf= z0Mg^sbF9{O(BA(Ha~qo*qjk3@)*Gn(S8mb7XP65U9TA&qdMoYq-+P?s@~>^jeY5DKe7*F! zJ_$){|NH&BQ^ABy_NEex=ifE7NY&$C>OXncI%MY7qp}`D+VLH5`|g1{C1I)@*m9<+ zLy=OD`yYeHOM4G+7Mn)TmMzBwaYiGn-l74#Rr(x^Nq^F-KN)^$;Q`R+*PVQTzqUR1 z_iMW{AgDCIs|4WGlX*dO%5@kD6G_mEowj6xWIX3HgrE7k4ZHLdVN&F|6ST~g?L-wJ94bnwWi2%pWqBsW^vOoVV}U1c>@T>r zL)?ZrcH?bEA`PQo{L7E?xvACBaN4?wSb=@`%tBDMsc9AFP9&5XvORrQW_UAf`&@sa z7TsvA1N8p%YXyoDW-$dK7^7^a4u9MHK}nGLWuNiJIz^GOxHgWeqYYtVbj8ikrCs8RiD!m#0+)7(7UAmii7g#QKBI2CGrpzCyVv=?s6CwoEvogM zAA=8^_rE1@BcH=tcW2I<Q|~DRc)9V@_O6;qlh(^hEXVhrzS;dLTIJt2G!ezsZFS#+aMrbx z-Qr>~P*xO-S-c;LFg}%2Pr`4awWMsu$!!(N!X^RJK7%qxf>hr)lhfTW9(#^GkN)Tz;sn=X>U>Zb;oCQ)NL4w2h;=+#{ygvz?V6XmlZ!7Fu5WyZpT$#>x9kIP;~^q;>RhV)!~^`?WzJQyTkX;%9=yt z+K?xGtCGZ7nc8HVC2<)j{(0_!thw3dWyyt2mCFmre1cdPf;cmO%i)xgn(Oih1WY9z zp<9r=zpre+-dbOf;qG5P8V;#WY#!Q*iTvEM>6m?4d}`r3)FQE}^?EA)&CAQ?>e!h- zWVKRV8XxN0PEg>_lKTRkR*2DG>)8uO(86y*^MV6XccX+PqpH83Hh*HYPWqGkSEjGr z@+r+47>7k@EZ*N#W9ZIWJtU72M)nOD3TWz`z7pgyUz4vlP5nTSx?>o;ny5D!@&a0q zs#edEVQuVj@z4nShTuDFMaX`XVWnT-J$Mcl+~~MnXZZWwJzwv-Stum`(q|~3yOH|b z1@}FB>WYAb&%{_xiGf}8{S$t(--mI}$hp3<6YoD|H7RB3)_sfADO;>Y^OYA7V63XU z?Qk-!e#U*irMIll&|8FJz)QBaoW62a>s9xm;Pv8#qalP9XZnKBB*_VTpwDohTJ3Lr zWddFFDfx~WjA-$7fn&M3YA1yhB*-CV>mRo}a~Q9a&+BK~2cfCws@DWxu56snZ@j0E zt3R(?{B5CvrsNx@g$TXwt(9brh;p0%XhNDNoLnW(3HWufD&#=H7e;B;eK>Zp-V~6*>g-x*89?>n6KU`F;1Xgj#8TIzTi7aot$T zzKR``-63V~3I~wh5fcvC=blf}!}H^Ii3f7p55&p~)NSnPd^IA^pG)OcV7Z)u0l`uP zk89^WElZT=KD;W@LGBbZHx@#1KYutL5Q72o5PEr>ESqt_uM<({?AB};MU_u}ZGtP3=Mz5t zrLx$FEBOdUSJ`l@+W2dWy`<#oR=U%DL_lc}Wb)0*?!1+b9H_YrW2r@&YU_q8gurGB zU5E^&>=KU60TbN4fsGt6M_}BF8@o*;PUbaU=9bVGrMBxIKy9Dfzm>Lanp$aQ$rkA; zB-E?12s9W>O66->Ou(D8gN73M7MQKP^_Hw2tzRB$8q_ei5t8X*J)*E;x#;c>$0gFI zCcf9uYovIuII1830-%F8wo`-M--}OKS?^Gzuk|45~eIe5*GsbD!1u_GjBj z;ZTv|ZS8THx#W|jFa=#}I%Gg@Amh08s`sYCtxDR!y{cB%-NR9J%U3#9<(wZ|^Pt5z$87edBVVJWVtjKIl4DKja&Xq56_s3YYI zongaA1E_76)$JP7#AOp69TPb!wCl2$*kWJ%)sUpy@f*%c3)(SO>s<>j)?trqBN>P~ z0#>eY(*LD+xy;sFwNZ($1vf<{W|%bL`13$whBtECe zId@%Wa88tp%Vnnw>Go)+z08jq6E6-(Wb9GaG3*vpCDA<*mU_$OBZ+z!{i5i{OGWR& zIYhW9by&Y-BWT)OILDT{jVKge4ziirE6CHHd_B8+uUAvTtpGWf)+7;_M3MZKZVh?w zq|6o;GqXVFR>DgyST=Dq ztdX-8bd(WWb_gC-L^CCxB6xe|znqkgKi6tnz4lr9BW9KmbQ_2QtCW+@Tf?)Tv^Ag_}4EF=m8Qzy1QRQuj<}0VS*Q+q_x3d zGUD!0>SKNV^L6BJxE09ex|WtyCOKE7yv=jh6ZCO#TP_bajA>0(wjGboqJO}VheOP1 zuEtLc89Iv<-Zi)8#iNuRgT@Pmfm(pXOx0bhTo#-1X^+^{FUd2*21PRWna*gdTeuz^ zx!*n7w5Y3e+~Y)boK4!+#~JC0wh+lG;*=N7>co|0?gQM~q{?pH1|EBH4TnQ5hvkrC zpB<>GY{3f?K7jZo^dM;H%;d1O3U8L#AAFu{+Dv$*GGbA z=KO}W*2QSh0Dmk;>`Qoqj`){2Pv<5dnbh_-0HH1?Eo4%V56B$ifFU9&y%=LNzFtR9 zvoyidwVf;+6Y7JCy_5-i*U!`;%xM1zQ$5We73|2ZHmRb)a$I%-oxQ@)^1Y@_YW0|M z&jzTkf(y8XkJkE6!Va}Alu6Ey_X2eENJT9m5*U94Eji*B{`LXxjPyqpt0mv?kwx#d zQF7h@A~jI>NmI|p%Jxpb0EEI-Ea#E;JUg(u62Mg_AS-yy^%CtYCN+Eu?Ax?(Q+v3T zqW9}Yw+Z$KJlcIr|A~>Ss=wL$)ryf`fLB@?wQI?dEn7;_US0~u#;WN*@tDq?eJ50H zS+N3-0yI-j#Q_P(iO1g*;YAo>RKuOfh4|RvHg)qZiIj-#ZosYG&=r%vB-O}R`}g$XJNjWlN*`}pRWTnJ_)*McQPvnoxE}Tb}G=N221y5fyJ&+I*-|D5?zIfJkRxo9x>nC~GAr^1b=!Jk&~Sq$-jf^?U1z`~0_{#{j; z2TUbvGa{6 z^^>u7i~Q6h4(Ek%myg4TcsKR)GLHTFfIb>lUTBGSe`f&6KK4?5`c=?=Axspo%6aYv z-VzuDZHgF86Ol12sidW3 zG9;QjhviUBwPvYV%Ah?rAtred572HWSWg z68KCzu}(M7CfN2Su{Vcc#?<54u)k<(1=9r2y0n_*>2xhMuiNhAe#zk=IcQ9Dq}#w( zRMAO%wcpqiD1Dd1^Pwi~j2+fWRS|5s?hZx2Xb*nw^Eb2r4KCx0MlU29M?Vi=b}JS2 zoxw1wCQ}r)FFMhS6nfH^S2A}57{i?z)znLWqENFqNDW~u6ITEDQ8p}V?9P&ecSFE@ zDgMxH8>p`GJ&l3MCO|pUTRK(f4S-l7CQAC^+gZ~#DG3dYXrKlmq~G_2Zb`0y0uEtl zQkojRF>?pKN2H3BqNcFq8! z*JzQ?0g2Vo^dw;nACenrR($BKW;H$Qu?6R*qfh(U($(1vpt!!Xq-AMnK0 z04b}5Dm1mV)C%s_m0})r-f-1M7uTI3R}SZGAWE$tPQYlGQjX(|0P!hL2KT%aVy<~Q z8x2!v4eH%(8jPWe4Lu{r2VIEqPHZ6j2^xUvgf+Rz3A3k9to1Er`*IDWo;94|MM@5b z7k@vwU_(Z>)5(`#rQpiCu3oF0I4&r{Ln>XiEBWeL)B*xF5xfr=pAU`>UrP%?b z8t0Q?W8s4j`!GFq&!lb<0U1etty8eer?S3IqTWWIicZQO>l6qS&u=98E%VsNy6qwk zq~my5c93&Wle)Ign=$+-Kapgevr#e_U6S+(LYNZvo11s{`t*r3fPM(SvExZV(>4Db z`=O7GtIZ(5U)pQ_!6!1f5&dujyj=cLelj;v|M+1+>9TDT^xTeqaokWwd5=OP=~#p~ zDe#iD7pLduGA&ORH}kEO>1c*ZzI8T{oRMt8s}ThinztBfdEITX%e23u!26m8-gec? zn`#=(^O=RZZ_Atwld5R{?cIufhx`?9kJ_Qow95L#_WH3e7Bi`F4M}YkW=H3w8FAMU_@s9{Tcb@9HoMt4sxUr!D5;*BXrAXmwn|qhllZ4iW6joukYhE8DKzRzR+8O-&IugJ>=bakCjvXSd1i)aziJX83$(lBdnj{Zu;^ zKmEs{H^p(E4!x<+$eFrQ+nz@Xk#g>G{blTtnVEOrp$EtZ=QMvi>~O>NjCZ4i$9!vI zd)%-OI??Kt0qJg4JXCuIbGX*Fx*nr=M;;i7?vSs7$g8FH>ho4I*7}O3ua;V&^t$J8SeqE2~W8;#|RTGH~7-}4T zT)A2ID9N%6ivCE)6$l@6UC+Pft~P|>H1sc%0X znr!M0WQ=(-SwSZNkuVeX^^qW>b!xSdq%;}LoTNC3=BxQSehW%KCGV#33SiV09jxEx zsGTLzImmo5;ym+7cIbJgSZ40rNzR=8CmF_Q4L&70tk{ru@C<9{QJSb>3eD=k-ERI> z6)71(r~H{}*b5^5H19%l$J+cR`+1czyPcj~IK>`QyJs_bB|W(<@wos{k`rV#1tM!!bEzbM-HJeq&`YhNQCysB9ZL-!k!4+h7Mhnny> z)|IN3wrws^YaOLol=DKh+^hcB-$-!&wrRx&gW6u?v3!l7yR6Ut4mW@U0&hE_?R0an zJs+c9i+eyQK?&OQ!D|#Jrj5g}&awf}P103akxh_<2uyUkBM*1wY@@z}l8g)0)B9~eM< z^SZ7-DG;S|AhabX`BiTaKO*M$VA-)c746$4{?x zUMmwPH{a9WUjoeMFxq}~QN7sCovZDpWqOTmvYFB-9*{geilOwCu>_^=`z2LZm+PuMir1A$W@yqLuOZBc7GvrQZdSqs7Ia(r0ntZ9~K2Q)J#3mUdq zniD$Zk8MjtRYaKs+x_E>^UOEqp>NVZrCPMKaW>9q`CczkRc!7zkvU%AKI|&9*0t&b z1Gf3<1@p#HD{8~j(7bG?URSZvOxW0)cVqGxZsr6sR?VxD1)3e;7Y6PS5AeRFHY->v z&-Z1uZ%pwVa9}Gwf4d|dsJ!bGH=k)b<=-nP z-vseB#I24t>Pk&Lz(e$&az}^?tiO~lo-%E91OS3CE6a-mYTkR3nk=E!h#&Hg&VsSB z?s1%M6oUZ+;B}bJOr}VeYQTx~@Rgp8??b2bzHAQG51MS~^=f*=PV98x>busa79yz? z!Qv)6JX}42T2XQ>hSd=~FOyi_B5{*p@LZj`!nepj#hnBq>~bly>N20hn!bC#+&5** zR+IV{zDyj-YogzMPi(ZVZVP!`KYL=sZa|`P2nLu?Z0btM!tR^*Oev2-OKQ*br6Wux z?4Eph^&#>mum>{`(|MNMzpPn7al2=ci032OT>SMKg0{n&ZEy_6tkN(?)FrFK1!9|f zD`jOFh{`$s0S_l~xT`!i5tj{id3*U~$PlrEsq&1|>jx-}Ph`R#Be+tR6Ki`!JVj&X z<~CYZ4=HlnsF_h=Bkdn(AOwQDH2#4ocev|OPxJeDVDq#uG*o^+?|5Bp7N2NmL-nw} z&(ZXEp%CY#WcXWI#Dn^mD3DL6B=k&*R#_Nk*(Ew=O4Wk$3WKX>?Qck;2A{l}9CRHwGPQ{?`40c1=?bIBvMk}j;Z5dS#$ZlIZF0C4T@g%({9jnEh$|WGD@>H(kENZ+lp9-0$dAA{d8Oz3KlgYaaZk@}Q3f zgS68wyA_UDea2nDSXSeL@-}a7TCKz5bJ7H8?Y2?-F0+jTB@-3S-(9aB z2ZVzzkYI~86V~+G^+v1d!k__-N$V&O!cxEGX`rS45^%Gk+k;0#h%Qa_U-Y9ax>QFm zcYfNBlV4c}L~Xjubs_-)`N!UejaV6l@U;xPJCsehXw*=I0TiLQl!a>xbidsI+{&`w zjDU=Y*aeqpc)kYk3_BgI7@Xc#`v*Wdv8r)#W#Fl}B~Y>BRC{X2sZ-|V1)Oie8;UfO zF!I=@WHSS^xB!jTjtI3}C}+ z=v{hP41L?IsMURZ^A)~axSCuysO{4u-pc4-b@epwDIR?#A&#Eo!xymvviy}EO#BhxY}j3^>W(-x@8;|06bLpvDQMk8@N*hB0o|kX zuRAWgAL|l2R;p@#Nw-4As%$cRCb<;P;0?6obf*gRVR>&Vs?SgAo_tTVYY}&SpBVDl zp&aerC%6BDeHCtf>kn98bIgm1$vLAVj7vA4b!t$i?-N`r2H-e8(5xB{)&FwezfC0? z_H?vW2+bdVuo_GlGc6_c%Gz7Sgid%4r?=cxC|hVW{X=baKsZ#VsVSi+#4tw7C3+HG zqD7VGL8<`*dbj(5N|-2B+yUpEYOS`;Mf~?M<}51I*DgP^XgT{SA`=o(#XPqwPqygl zWUZ9hkYGuiF}#~0CDLKC<1mRIltA70G+wL#d!fS_&df(-!-y_nS>RQT)kd#Um*K!} zYihi)4v-GY zkbN$0M8!LPJ#)g| zc!q}OdlHpz(~xBn>&d>KSCkq;HP(d|Ka~A)z`==MbES4ruQj)(9e=*r+3vao(gFSC!_&^S|p8^4)mxKvgVEh z4z=z<+(F*!r8+w6pJYm770=Vkab2IPmzQHU<~fN$#M#YTLf{kgK z;mKjxk+AGqOC2V;e#CJDAJP0Ju8KhSEb{NB9a~!nNtBZSYTBbBi{O8<4$uqw6u_MB<+Qs{8@>o9{+9g9&EPAj5JM>F%?wr&(*V2gE=~V!g9Uoji1yJ@m zuB`Ux_63Z|?uevg$L)i0R)ad+?;j~nn_{vqF-IZUK)bVkpEK*xc>($90XoN8ADeg`;Ty4I|eI~u&07A7|gS~C7 zKC|zR@SseMrO#SlIRJPN3@ahEn`SRc#&10?#6d?jeswchw^NrI!UTS&G)0y8;L!>zYFM31h8FcW_9IUaxxA@iqtB5Q$F!q*#P>#*kP1^3igcShxoLl=-4h%A1TGVq zUxlrYF+;bh_T0;>=~XFPp?)RbWX<_z@kUinjF11dg#b>bbmB-_BtW{n1nti@(3*{c zIk;LS%l23Y$-t7JN|Txo|5O+l%E|i6%3W9Q{sX(7{&YkhXQ)Bl*v^?l4(^iZn7Wo- z2Zm@XHq52ALLYxbzFK*RaN)JVmZ^HW81B-2mmGCV?R%58;w}I2@<8ENIX{7t@RMpO zs=6*9Ol5WL^K@|3`}J7Ou=F2#J^X7-cGsa`-lSZrg1b$Bqfn%%`~ zyutns1Qs;&TSKF%N`4d>6F$=F0S^98-Uj#<@XsJFdB8ktp2Xn<@y8g$nsx2JePqRc zal)w^oMU{%;IbIHjJn0Y>#|Msbe1-}J6y0+n1_ZyzaEfPtzPO`<=)pD(?Kfs+NnvU zHEPa+7IV9$OrIiJgU5qM)Cq9^+Ph2*hP1K-bB>b(2;UlL5r;t=I8qharCtpPC~I*5 z-d_1n82@ia&07bih4B4sj0%3oVypw$5MGjOho?est}ME z3zt!1YAu+*VfRq@p{LTQF~K#esK*JIPm*158O7Fn_Xp78srF4Pd%*vRt^e&9-aVqk z@($a{H*|8!1~%k_1}Pzs{Ig()Nwtc~V&Mu6OJI^N^{a_*cXYhttIhoDiF0MA?9Q6> zSZH)XtX$b)p#$x!UFK$}Hbx|nr9^hF7yG}v-bnifL7Kv3ei_z7kaLHR&y$uxHG`ek z#kcLe$5-P3DKf0^`(`lp62c7Frw_W4{Sr66yTH1h5=1;>bOAM<$)ZuQt?^UtQlj^G zp_B<2H|`*#015(h$7WK_ZoCMltyF}2pvd+jox6*&S5|C@zxe6%nVLbVwzVxr&I z1M*Yf+IN2+Sk>N~fDPZg-w*xWeJnZ5vJT=$MxNNYU%CSS1tS37_8jwxZKL)glAaQV z1z}cw{Y_!c^seBARs3c|G?Xh<^B*k{ua15b+A5mgW}n9_3$%31uZeVh?pwzf&FxZS05_XEY;_?HdWGv1I}pHUWh? z3`Tb{cMeaM6b@jq-GA`e9uy#2UHq-!iycSRSCb98~ea|JcjnCMG( zXze%AN;P@<98lwsUkoffG4q{&}@IE3mY#Zd7E- zA~93^pawHcy>u*SF4%<}zUKAMt@ikSL=ZXl-X}B6psBmIr&=&eE<*~Z>;*+WMcK>r(cEQoQr0u01Dyeg1 z90P{G_#VpQOl%`boHqQeFui*8*OL)r(XmZw1ExTMm>KC*^x&#vVDGhIH$1m^*}g1l z&!Ad0W`jq6G}@?yK@lfl9RL$u*s}g6_-<#>S}frWu4X$~i<4fKyg${b;%7JV+-h2G z?Fm7I(D}A$wXxM{oE27d5~$$^I)cA&DJ*HFgeGoCR(Gmc()<|lH|!&_C-$q;pw|_E z3Izbmk5KOKRhmYDxX%a0AjoUgtR|>;&a_C_ZbfD&=6fSRL2f%OhAMx+vzSi|_c~H3 zrWj$L7~^w2I$oAL5_`qQCEHTwW{MFQ0YOwvWPGM1fpRQNE3pCI98%0gJH6*VD(t@v zU#fGM;&dv?Frhlt-P)(_FGX}TbkEq%d0CEHmdq~|=a|yUOG-QcoV`L*(YqCXRYug~ z`~UFf99UX|uy*6}eYRwP`e2|xYKlDULiJVq7DXZd-u+h(Mo(K?$CtYiL`ZjPyR~$V z`2y47_1eI$kWZG>Su_RE8*QPr{)e9z=&fcp_1SX$IB2OIfAblsWOmC&_I22E*<>pL~aKfxl;iVD=sK6LWkE~XM? z$5KYS7-3qAmGoOwgV#-DD^IcP9lCq4sMDMj4vB?$mIm7d%T%FELl45YzGNoM}=F0ElZ;^O#czXn(8jq4u3q5Yd+;Hc>M zaCMitMPg;G!D2LU4{hg#ncy_tdLU4p$hE(!cE~2tHof2SW53}yifL^R$*49h+Q8$E zA_a1WlZSlWvKxb6;W!Vneks-2sakq;j>VjH4h@P7ox}C;-|T?dQU?meTRBNLp@kdG zXmF(NC#~E%&tN;{Uei=@Xkqd$g_=!ie9x?J7cjvn?6?*ykoK21!Z>ISivj3<*(76? z^>;IgCBiwqKjb5rt@R23n?QD=(&GgnYlErkKAElS1g$_J8x{*H!x%cLS|+NAJ)UacJ0} zR1r16g{R9gJ^Gjf;mk`yL4}Bp&n$08pNwsxUi2Q|DEjK+Q%7uB zIGDctLnLHc)o{m~4PbIA>7m9par9wrUHfXmD ztG8~%i7Gnj|MP-b^8%R+^B?YYdkgplA5O>Cu$E+2O568V)w&d%7f`O+j<-xCwtvF( zTW&RUs6M&EC}M;=`q45fP5{R&vJPn|RzjcT#|~73X-*Q!j{NWV{?QUU(RZ$XpYC0y47u9 ztjZ59oO;s#iQs|&-%lyLxz(0eh%meCKiD&oQQaB>JQ(qK+!r}5P*UVn{a`|VR?0!i zDA#Y*QyRdE#-oA@Sro`@EF3ct;xN-8GpYznH;_svK+ZY-Dua7wz-vrH2CV@k3)}H} z6ZwC2_vZ0X@9qD%>aOmS%3TSiat?`XLy7F2ii(gWYfMKWge=*{)@g5&?Ab+Q$#$|f z7+Q$PHe-pgWEqTQWGrKh<#)Y@&N-b=_viQc{`vj>MSCT8M zr17;f>wr*%aOyYd^y}@ylqaR3pLA2awYd-bIXj(4&u`9*n3EoOx%)ndRp5Rl7^PMW=)f4DA~c>_v_$A z391+<_TQue=mlPM?A5E*79LXNoduM{2nEBR?A7@537?1QBTLxB6VfuBpYI5F@wTuMf>Ki$;q0=Gz;L z(nfL#**c36yUc$38M22OCw$H;+N~32bA~;Xp5PBUqYE+2VKO2h2|2liUFLSfJt-Wx z=};8I5;f_7$oc}xPn=RrgnvHn3VRE?6%%?r>&cC$n+r!)7ct)rdk@5|EN0o5d&hziNHhs)hA;v9pfsWmLW! zZmS(XLhXN-s3-15zgW!U1V9k`gHEKp_Hn7xk8;az95 zDG4E%Q$mmwW%Da62w3R@!Uqpjx=USRwlo>7J~Z z={X*fV?H%=Dd-)x(ov9^oa;Ij81mcUHm818LT01>_*&N$+^L~*twxY*7q20w!t8~7J?WjdMH+T-E&8`oK^vaARU zF=;TZ0^)is{=2*%HO8!0g1AAoIHrDSdC^PnI8RU~*3d1HsTbIQZ|_08rl9Pq33R!h z4@N)P91}0Hd`M-gi<5%!#v%eB#?^W3VU)*yfv$^+8Z|oul)JuH`>y|SN=^MzFiW|R z;iplwM2WC#?cPo;>0YD~)H7+Cwl+XGhj?sECew@I7!DMq)pIGyNS4ugK>4VW_m{1Y z&qS1uR~3@sP^hVao6yCJY||||s}Z@Z(Wg$YGQ0X~N*}#An4UvVdhPy)->2wqidE$D zqt~F)9h7P`>I#y$b~<^h|Ed6euL~*~8fU2immM@#`KP|NSPPP>@nS}`pJD1A#^Lxd z*wyn!^d(A1fEXib+k{WoU_p)8bRB_&_&}mrPja*{QPP6-HAuaj-?e%;eI!e5#fMUu zQ^$|`OSIe4g~);+*hSPjk+^2_M4n;fEl$nH{~mX!w2ZXrO8aw8jSz3$7~umKrb$z% zmLS?I$LFTmIUQQM5N{Q0{WD{a&}>_mvu`Kl)t;}~ZAa+DS*yG8BQk z*wcp__(y)<1((H7vOL84XZ1_r&U!~aJSa_OrGGj>JE}2Sm44E|MTs370{LkBE2Zaz z76BLm5*p#Um%w(1THIX5V8*G5y3U^fcVx_UYMX!-Qep}M&l73}D8>;0ob#_$rH#*j zL5z{y5^JfR5v8~4)Oy?Gwz$#QJqn|y#LNVB3?^pLkU00T;55FAO#d1N>8emT<~>lA z0x1pOS$MbmM9?(9tFiA%%eEHL3GvKadSUCD7am5flJvwuW!Xg^bgn042SOAfe@%Tj zvK&V#DXP%cB=_Fh)i}|)+%vK!lG(xNZ$S{++ppl+7SjCNg3l!S9FlysAP0oDay?(e zhcfGV)1tHV&JH^bkG&PXX6_LJQAHIWqtIaQ9!(pgf585!oUg{8!1Q_FnC_P2boOr> zFm2Y3VcvGQZtjDw{`bo$%Zsp$c|)pdNGV7dan5sMKO;@SZTRNjRp)PBZnnm`=6PG_ zY-6ReeYrjUUmxBr^`G>@wBra^z7UOJ#C7G<`)FFtIKgX(nYz@#AH0^N_RVAYcVA9w z5LZl--{TuoK?2%t!(xmVOz$a*XIgL;;WxWd)>kt!QY1;d>FxHv+uf7sxCkCoYb+7N z;^mc!wP}21l_RU>sgmcl&=kWTQB>+Yd0Y_`T;ozH=OrsX z*tjoQ=AFPCmG#eM)jO>S;OQ+I*yO=7`^O10xS0>vvG!-DT`AdGIN|nZ z0p*0Ey-+0$grOngzh8O-5 z*YCenp6uyOTxrbdRr85DA{bQT>3N_zZ+hY`;*9%mXIVgLEcIl&Vcg<9b@wKwC19M* z?6ceAol%wGMgRVe|IM!{8PG#m&6Q)tBin6zdh{ccq#7M*Lby3h=A5ee?;CZkF41!m z8=D(%>0R&5crn#sw@8Oryu1QZn#RXTCI(mw_j~$qM+w&U#J^RkESC%$Q+S zwfdX?mu2u;*A2^eixYCo>}hN{SqPK(xsmX7vO9B7<3RMXdrW(Bugk!m7D-J1k1wL~ zxL?0%cjl1OF$lRA%Ac=~SeV9U)TMG|6`ivyeRRkk~X|oZz zWX2NhTX~2okPp=%c9VG;FJjhmVU}JLSj78GO%nO@hk*$9PHkPwi}}H2{5mz8&CuqI zNE=FUp%CpFx;)*1Bmc2u&slaU)AM8UVvf4lpwD930Yl8lWq*DoBYp%bNFi0uC-{O5GkBVW`v$=<| z)iBPf11$1M>U)QJfkIw9wfsJ*%X_*c$06S95ES0r`mu-DQbSuAC8_V<%k^P|(7U*N z#jc3@W?MZ{9*V?viLo2okBM)NpTiY=e!%3k+xj7$@SnyP|9yR^+X_CDWe#elW}gOD zJobjr{H{04Iw$Dep1t~kb0tAeVWiZqFCrjrd)z!FGCOebQu~0v{n9LnRZDKdSClH$ zCmbyvD`|UVYfwBMbm50_kN1t2ANRx}u54?$ODKYK6_XV!%G|ED#0kP(W2!Y2iY=}# z%mlYh`m0>w?yhH`qKAr-r0at_Ss&zBxBvp(zMYcI+nDM!rF|MQ7=Iw>=Jz~q#>l+9 z{wW*QD*?bt@W$Nfr>he5pG?(0z4S_2k4?vt8*ZIVo#Yvoz?#Ur>b{>`O>xG5ztT8~e{#o-g!WV6I%J z*5J>q6kRa8*%(1&iECye*&^}Ytwc$2e)2Icb(AG1*dLA^|mDNb$HDv)NQM69O>s* zjRddl0BzWn=?J4Z4Z#i|2)w4;65eF(f_?T9v$!!) z8v=#54-AK6Pk;1myO_9_F2sbSiBP+EKTL27)~##8jo40n%%k(Ret0$~u|BHEE7P}o z;`2vpD^hajbfYRCJgS0!ONXKTDf`!DP8^eC2f22=@|h6NEw*t9O0G=ayIE&ZbKyc2 zb6!x!f>F>b6}Eh=vf(2m7FN}^(D8_%NvJ7d1*oYgFD*gN&+X!v8~!_jbRkcvz44ln z{rz66fevs*JpX6hfzA5^SporKh}PN7`rSw>r`b3!baL0Lz*ALc%VG7QMGygYC;4fm z;%`gW{;|W`{sV^e>@fO#ol}`pJEOg{g`79WDxE}Ox7Ui+`u3LvFq3co!P{JewT0LX zzxa%x@!7u!!J zqf!qfdQOOt%vjvPL@d!25;;Pa7hAqOX{p;ua;eBCJ)mvWWru2Dhk@mBO-YPJ66Sv> zVNQ5WQH)6e!#wQ|sCd*OXjbhrQhu9+^d4*D`$^P*A`2l&!&fCvkzNgIYLjkM-2e-( zrZZ*M5In1hN?-(YF?Dh`9?7e89{{WeRTYI4!j@W^I2Q60y?>;LpQhx8!Phen{- zHBBc@ujW2Y%;t=BY9UpIYR`Tsijn}ygTI&HLrZHztEntA7&iedy7XW6X{S3pnkUr~ zid6f}ska?wEbWRbDn%HA7$ko#dYk8j>WbxK zFU%}rf00VJ+P8Q4u7tvZ)e_stD=g|2wMD%|l4oqlZclSzHj0q5)4a(-brz$*PbQ}E z*EGMw5UekrMNlxJ zW3ZqX(q?*+3s&Xb!9rNs)70|uXI1lFd z2!&*i$qrv;iHyCt%Of3Blw0}eou1F~wU#Z^r)WPJ?Xtm@!kI4r`m+@8Ij}`yEFA{a z7Ao_hYPFu``;l;0AY4#TeUD$=(eBllO}nXa?=v1VyJd8nP;b}d3)xXFU0$RzH!n-2 zFFD7r+FU4rp+@PFwWW2@6E^;g)7#akiyhQpEL)A{+0Df@NX{>4atFwFmtHODOW=}{ z=F)XejPT-*9-}Q>UYIXY)0`Av$ujP8JP+wk$mK;(eHx2&>|$dgqsG()p22RI0+cju zsTv^SlOW*C6QH*=w>_nV>w}N2~#Nnm=7*L zHJ#db<+^Xvoq`)Q?w^OztcOLHx-Z%{0-#Uma<7K7WYQFZcmPO7`4eip`a9Ys$$tnT z0LOO4bt{JReTM~iGCK;YN#IRtztZdSu0^~K!yUL#`IZrRMR#N0(xJTqTmG(kfIVhUy(wxbDtxA1xYt6^8K@)nM1R)$}gmzao{gw*8kC zHO45qM*B@~HomeO(#j2oTJNB#X}5q_)slwoDs%Ft{lOtfNkZpr-sUX~W5FW@p}Tz& z$x;n<9r;aI3014if|ZQn)F?|oLfPhZqKNg8n(Ih#HcKx~ zew86wE-~IaI7T^@uY|~YT4JP<^bVI}?>%#;7OJTJoPYXbF?G=Hu0WpP5#_vk@8VS` zaY&sH*vD}#a-Nfgk}4DP8#<#l0cC1=pZAh+xf3qu%@}vEB&R5G*6DjE0kV7Jk;s*f zpQu9o^`ndXe@jc#buMx48!7WO>KXXzK3Y|wwc|Qe@XZ&IJ2+5-ON2 z3u9^KL0`sR8}eX6+;Z`tCK;u*ueee`g4~Wr2OscZx!q-% zhV;dWe(hg+wa`2qVKZBKw#?*fYRpbVA@@AzuyXy9qC34sKbrBb!cwXtWnBy^bwmnA z!;F`%>fgir*}x=MZ%iY>!K^a?$9K!4%JU-;_*R^qw@j$*&nXM^ zhqo!mJ<;b^up4fP9(c5!`}B(v1lF905lfA$dQ>!JDQ?D*k}E-YD`kfynAlq0JcR&q z)J}Q?Ht<$U%J;CE7T--K4QQtE)@$vL1QgvqP%iH z)Rc)%KW^A?FNL*ygfLTG`(C%=4;m4b$XCeC8iGo}gF5CrS(7!tXmDuq9~7SulyQ_R zBFzt6eq4r~DDDjLRR0*|1$_O!J2~-y_H=>Q34SmRu~e3t%0bBj;!rm$Pe6d)Ju|Mh zB1B-?OeUW*i`|tfWga6JzjPgeS)>lJ1faUPeypTCzsmD-q}5($Zhx5wdCfa~yTg|_ zUTVkf5(#*^m|`G>+W+9$sQ4+#4(`r*VdO5ndxAX4Xrl0Oz z>#a;*bZo51Zf}%WeipydF?6O?y`L)}NGeWzSH6o;$G}$_U8;zo<+OOkj+ID_^Lq0i zo?PAvSQyt71myUyDVPE_KpMK|4_OEdBPWSH| zKl>m<#--^$!*uNJ2PV&tvigpRb)3r+nu_|Au=)KoS87(I2&X3Z2ldUp9eA4H6sJkE zoQba0zpfj7slo_rRDZyTG>B|Gfq4)TAn!2|n+t79XzQh)I067yh55zx+hyw_&ukwG zQ&SbIZNux`^}~_U4h)}pI~!qgB6(!?;c4nZ$E6)n)*%-&m2PV1?+m!qs99RSUFnr9 zLaEObS*#q#_8O%?W<%-R&`ijqb?1~05PaJw(UY{2BF`PdLG?&psLzu8`AMh))hlW6 zrBY;DRB`=E2k3zD!_6myliseJrx&on0$;X~OnQ*h+kBqn zXk@jQ7I{BF2@c7ebYo!AXV_x;{u9d`#cEUWBw;1#hAnT-W-ad7tPo<)%e-bpx7Lf? zxO7)JRBeZ6z+ticP6Zjwyb)H7NG&NDK^IB1t@%>DVsBnOmr32I$0+(pblW z-u3xk6-#+kouJxq_Lx?A5_V9lL?X3neSuEgeblDQ5?!-<7;Vd#Fj*gBR7no2ILCZO zD*Z)KTmaaYX#oZP!RieqX?@%K)by)7b$91YFI#^U*y?=TXZfdmq55Y#B~A%-w0u5W zo`tRC&bm(4|GciL;nAaaHKxYhPq#Pv6-{^(73^478bz(cgmkpfT~mL|+};1-#Y(5s z`Gpp_>FMU|?b-sL7oX~@mQkJ)w&2yB@_w@Q(GJ{b74I2mPF(Ap znG7L-QrwWuyXPh8hIm!xiLi@2zw3P&FURO?a_@k2G>uz3V4&&jp zIxdBk7^0(4RQJ>k>X;BI(b#OAv@$?4+Lrj=S1zlzF9~6T=k-Ugpb);>ueqRe{;kUR z4C)00LWl#iiwL-6mHe_yeYBEnTA$G)oL=PORrr3y~_Dy(LRa9wS*A zNNJ7jbMaZwY`2};87mNKq&irjAMH7Kw>_ekz%Q`qPy*fpiI>po?a07gUajF0i>ndE z4OP5s-SA4o(j_1Np$h(w#!p3O%kwB^0N?AQ^VY(pScaN#Ch_d_5D2I4uRC+MLYRt+ zV^g((MPdbshRKrklhf$+b8gA7dnmd6+)C->Lmb_y@o?1avqStxH*y5F;hD>4H1mmc z@lg${gS{i9^6&)AuOE4-nJw6e8zPpj+5p25mqXsBJGJFk(qRTi64S}w%g1i-#4THN zM4JMAW~SMZ3l-s|c33x&Fw}DU=iEwQ0IY5EL1ZR{?*Jbuu&}WTDtM+a> zYWeKgxGhhm_(tutt_^`79yYcxQ>KOzIVWRT3C~o`%*y&_3RX;wap;J~Tee#yIou`OZXZ2W z7SrSoZ9PUNZc!Uw803^oI(kAVB_d>(W?~dM?)b5IcS<(3TqHpbWX)hHB2m+E$qsbLdYess37iY(4RMWMTBY03w1(@!Qke`j*Ow2#Ja<0G@lO5et`gA-h3pR9fz1i+G~6LN6GYKL-HXPtHPigEnN}WM#c3{{|WAPqZOJy`2guxE{-8hxBO8JAt6? z{eIoO=U#POeB|f;rh>Q4wy9w`Wx^_S#_;1$8#ol-IXHjPVY?V-Zx*M%vb|)e*w~Gr zNM_O$^nF931fg?9M)utr2&7be|}p;{BjDb*L^k(iuR!5u_62 zzt?akeMZkW1D11ASBfxj7g=*|pI4$EY(4fywT&)x>5Z)8klrV{+O44eg6T+AGGAlE zqA}A9`YA4~tzE;L&WyJEl$(a5c(A;w)9UO2Q&$IU5$E~v`RlWMl56RKuP zBKTajrB6ymF<{wW2AZ%U{WOfLoAiAR2Y#=_&AFtQeG+ z(46$l=?u@>)?(y{@cMwtzv-cCa3nMs+bSueubTmI-nVsExBK2bJBFH(nWzwt=E6XV zuGlQ}3VgcSq~e5aX~4o!yLh3-v*M2xThBtXz-!RL2^qgB8j&YUGX+{i^!?= z22g1y+QX7?ar=^W$>hVd)&HDdoor`&ma(OlkZy&tWHDK$ArscIUt{fG2p2Yp`5jZk zY=8DU(87MC0S7W#U07q|dHw@#9(G%3?*jsId=JAu*{vR8lHd8)<3lsi@gHzkPy`u* z-F7LsZ?da&!si>{zrZXQeTI(E7*nw-fW{SDe_Gq$*H#6Vapsfb{IsIW(9w45FYtSR zIyMKjF|aL0W(-Gj8&f$tjizX=;|P3ya&;zV&Ygb2IO{lbIFEa4-?$dvZrgYcS&^R` zFlhk+zatsQ5sa*zaocM+0;)MHKHMY0-6e)e&s-gH?YhTIcE!>TOUB5|MzSPhV1wV^ zSv_Z2UH`p*u9khJA3nc8(@s=XvA|3e;#SrdU=jK*SM#yzgbs?Sed{^iNWTEuy>}AN zC@LmohaYP*N%8&Z&6h{5H`MY;uic3e#U|T$VdjTncVza}68Ktq*Il~{*GNyXf=z(- zuc6h=W1t=6Q?v4t@=|rRY9x(X!XxzQeKAUw7ELBWtrm0)dTBhM2srC7w4x*NFp}UAd?Aw zg)hH0fahl;F?#-z>B`u{`7V9|oR)DuqZv4I93T=btc9N4yy*4P!J!USKTyV zn53*DRkJ>g!pto#2^9U~U<)7gaRDLw>j+u*b zHD#g;>mKFSx=M)C=mwVJX+<`}uj?AJqdsbuiuNoRIOE&fYu;*hIMM(;k1qV?M#^XC z)QR*H(TW3Rh1>PET^uR~TxTqdx~d+OkFQLVET=hL4WBHGCCvfXCmE2&{5$OrX}qnf$VYle5*E41StjC&XmlpjmZ!hF zg(UxxbahJ)E?rwRwM#(O&&L7O*qjb+dFg^3HD%YUIz<~#$}JsJtCut1SbXhykWmDt zBmMnZZcolF(uA7kmoY()r>V+o<1ATh>CqZVsxOQRS{TAz)unNflDN={rT zzZsO9{1(-{DT#=vc;!dv^D~ve=uvRdOr#|@NH*8FqWCkSfQM>JA zYEeq<7IMT~eaMP_#&hYQ8t!v}`OinMv}sW*JXzi)V4#HMkHfcH5#|si!8lnY=oDG& zMx}7RxqcOm#+k!OKUJ~x|0TF~hYwTzSaP7m&c^&)!JT|Q{$7K7s5RJ};v>M|F6=@! zZNwE-Tb6YY%e~t(V&i4>-itH}@23aV4Gr5&4wowLth&;Lj$EX?Y2xL)mgQDSHTnRQ zYmbQey%w~KPdmZ&K6EQJ$9ADeeztHvDHkO`|~KFk0DzuGBqAEFDkO`VM2d} z_Aw`f)?a@{WFN1N^1gd4e>gmKpodjSqG$m@jHbb75zDt=;+#R< z%6dM39sEL3LC2kGiDcM%*>HI8H5JaY^GuxOGVhzOp+TLc<4^8~BE4^}9fSpfrgkqM z#M{q)&NHWnJ7Fn%b=U$Ruy34|BC;20rXhQG?encjQaqxDu$!%DuWAo(ZX6qGClSFv z;rt8ec1edD*WN_&6<9*jtF@n>oc{<%6SJ!01@_g1UxI%k5RstT@;yOz z1Ci{JEAUX0*Q)&5dZun%R*JCjVH`|;nBb~15ZE`g3|8D^P{N2WvULWldVqA-c|7fj z`So^f$ItIUk3dcrX@vFH z7zkrTwEEf=AdqX@VgIIv(-L*-CNWk_G{>o4<1+^m*SDHDc}XNdL({cSZ+kZFwSdl{}>5x)z&u zA-WE`3oF=AtBY~1iswq30E@N95b4gUy{hKEx#9Xn!8k1}x8x$yG!pDkwsQ6UBO*7J zrkX{7a9}kD)~F#Y*ilLD&&`fGvqla5PpT6%d-+wGt@gr1KvzMUgh^j|%$zN7Z&7bo zVsADmdWgccl^4C=+t+6vRAKf?FRlH&Sw@%<%Zk6Mi7fWWw;xjn;q%#RbG^V=7*{|z z2>vi0`3@qW_vBk@CX#2L!Rg;iU|a2SeacAJo4`omEk_U!9(@Ir#g23WCef?J7a9$o zaDWN!fe9MMyUu3Yy`Qo*JqxsbO#v1Otw`^^-*v}>0iX5*!%mmtXZdI4Dca9g* zkw-2EANSgi$6lFSc-&dqh)2HnD_ry6p0Fmg?IwhDCXedTW}~2B>7}nOM|@venuUI+ zr|B=Cb?`wY+g^k^xg$L`d^}U6P$s*6Q<6c~)Mo?4ph{pX0|;kr11@kOR_A!`A7$q7 zl?{p?;ex-I3{H54D5>hQ$R|Phc)q+iGmX>p0*o`Xj6K(gEW%@`!Fy5%VCQbbReE0~ zT9&*sH1_Bj|UTbX#*pvcz%}yL0_$8lo{MhPdW#49868H@!6HwUnAYvt!+?9{WX2co9k! zca2?-Y5hX5Q>I1qqJK_p zlT<@)ahFrI$qs~OUH=-Ia}F+{U1=D~-5=4sYV`k{@eC@p&)=;gjEHzya|tq~h<);= zDY?sG9dxh-duxOYeg-k^_XLnKWomF>zi9tCpC+lJ`2(>@%XYRrT{J0dzzXAQ06?x!>D!)CRn}X)4Y@3{&BlWBSme7bv70 z%KdcqPLDaTJq|i0^tO9Bca;1W=V#`MD^!B`<#WdTm4@R1dZ0{*4Lwl7>Yh{_^#)Q5 zO6$rDfx=8+YTTtGZJRYYZN?QSc{AgmqH3e$o*>@*i;w>8XdSmT8eu#6NqXu^O z+i2WaYhUZZJ2NR$wm7KTyu>870cNNtV$W>gQvSoeRJ0}UdU3Z0l~y__)%)^ufgOps zT?yh#gQ5Mj7c&e?sHW<=hKnvp(mU47>%JX@|>QH5fj^$U+Gqp z!``Pq_flrxpcVm~gs&xw$WUy_cQY#JDVA}G0tvOb=~uD2Dq~ss#Qa88^YW|ZrujBE zc?0woF8Z?(w2mJ^l}lmDUD%uP@4rS2kH>%&LM@&!9JPkJyf!n3Y~szdI9JWYjb?K1 zq*6aM06e%a_4g{n-mEv7izVaqFkMyrSC?thO=Qi1uDRooCSnvpDOO4BGmjPgB>`fY9_4+;PXhw}t@Ya!Kwn_op z7m3n@e)Y+&QefYnQZHI?4EM?0w@_CkxjM#S{8CU&tjrY+#)K2#0j}e9gpoJ+vFYKd z9xZ}`-5H{hmL3Pa3C^S;Oo<(avGq_Cb5!grb04^^8=h;2AV+pk)Q2lGThPrs&>%L& z!ma+87K67q6M!f{QNHh*D}fH@*3RV4kYhT&TpP{@ym!<$`MH_`-9ujsre9!5e!4R% zcMJC)%kQd7dmNZRK9YtZ6^6k&!)r1)hgla2B6 zO?Q>^$l%xE;GkHVmWv)1(?9)E`xqJYa*s#>Tygz51JW^uicxv$^ek93X9lZS*}3Z{ z-fZBK@bjb0bOR<|q*O0p!E#p20m!1H1jD92rq$28`9a0gL|P`&^gA45%}jogcX~&L zaO-u0T9r=iEFTPr98ioEkNG*ZU(rmNNr<1-XLdeMfkj_!SAWPO-e*+1rR??c^ZN(H z&|tArENajIVzKNpZQyrG*=E^cjmVG#*?m~BH8sW&Oy^Q?Wi+OR0OL>Wm`Z2g~&;4va3 zYryjOHv9CmJ2K?hv)^POr|H~^L`86(H5=PK#@Fv@t*s>(=qTVCLnTt*=HQ27x+!Ln zq9X_V=lkf{KG;wxwJoSRXaDI~;uY(Smd*IteuwEQtAn*wY#veA@oy*^1uonrUdJn} zuKwt3VTMhtK23)29$A#&Qzgh}+TZ21VRvzTd5)PLy>+1w@&5kv>1MH4{Syix584#^ zXMU5{1cJRRs*jNV--zT|YE~23{hc>eh`cYM2r4d1nnXc-2LAHvKZz2mrbaIE__!zl zU7T7Y{J-{D=(G#(MoVt*Ey&A#|Knw&h`W}LTd;>;%j^cR9r#@LpQD>a^)hdCS`(|- z1N;W;{<|x2<=5ob*d0E4BYr;BDTra1a7efG6u8AXq(2>I)Y8?@>4gaDJHG5l4aacf z5r^LE-kxwHg_)>sp_$|+_Y8Ze9AZo4K)AF>Wj~V#o^1+S zz1@E3W{8p1$FW2ZpisMTaRpU-c9eoe^e!3iVp9 z8?G|N9G-jg=ARQiW!^keX42h;pN4+}N)?-z{Pi{WGLX9^h#`!G(fwtYvxq18^hWo@ z!>Ugl75&Gm#q@8(5GUfe-1N_N z-9{-HTRyHOJJed^c6iGiaRzUE1kd{iW5`^(zn}P~%r0!RtxLk|z9FP?e4!^_9aUb| zeus#s=jnOB3U%xP-!DgJ{CAu4vK0^yPu8VIYup>HsiR{}^<-d#LN(C71JU}#Ou@4Z}AFE zSXp<9P9>+yLUjg+pUEYn!Vu5TW!B1nhcF74B$vJXG0vfPhK3{ZM`skv<}_jt5>7m51_*x zqGMJq7RtdU&*hf)<18-;kgg9DxzLnhpd|V4c(h>(R9e49j6IxN{Y4(HB^;m&m}d!U zDLcs}OT|jX3f$ynB6%26b*YUXyubE4Tvf5k3`ycr%keg-GXr^17G1DV9TsTLH0v&wzYDZrJ~rhc!Nm9m9({#~W2kaC0h4 zLu?L?rgPt4uiJVwYj&82NgZZ8G&mFi(D!c=%wD<<7%E4S+duCZ{CGn0`@s-Zm?Mej zJHmoU0{nP^^Sg}xAACRTDdBnvsHCr>q2EFvA6||5#`VLWgPl9}qak7S zH`$^Nco9e8g*B?e8SKP$f+`VTzDPkuX%@{-N|la>>ZmdYiL)CHGx}Bg9sIxE7I5!X zrHRly7w4x4P6CXG?i|tB>v`vLa~~SAj>ktLtoi%RQ2YhDgG}hXr0l|(a@%xNu*4W* z*b)T%*RX+tB0J;~7M5aaxg#)z$~tK{M4dqB;jb<7(Dm26&TY-Kdo{JOSwBncy$Et*+yrS3BxvB?=l zfQ2>c7pHUXf1b!h+^tCakZ})n07)P|aCQD!kG8v4Dsgd198vZzdwo+5aPeX2xa5%q z-#qfHF|F#~6((^)PP;&ZS$tWf4mTRW*n>%je{fFmByiVPJX4SKor;X-%uRRCOzp}X zc2Qi_^57ePk!OJ$onlF;%?F$~02se0_1cC0QG`*V9q#Djr$?3hU)G&rQ+cV}unwJ2 z5qxU9!i(ygOo(wG>;Erb*;HcYbg&fBd=%%o$lYO95oEn=+52N^^2E2u$+K#S-gw#xb#6y04PqQ~1>hH1X*w@E&_9-|Vjh=k7B8lkN<wijKot{RKv7>RYw&59A! zxcW2DHVpX)ktvsxK+v>u%Ls1xj;H+joIWxiHn+ZLQ2Ai%!S9IK{t;iutrvf1o<`a` zgpNG@j-$OiY;}55=yx3YnEK>1YqGIHE_&le7Fm$_KLv zW9jh5oUHMpbFetWo~>0NVOg+H zj2}JP8)6^Un`+eUcD-}BGN1UHfz8$) zrojK-_EY#_+qh_rV72GcOvf#-jHEq?4kDPEUK!FC_WlXfIWg*f@kq z&0W08(C0Dro<2aXI{~g!j;5`Mg0jQFaK5Ig_FS0%MX|5Rugx*-dN?8t9MkPrl&)Qc z$2elxHl)>~+MlUiadONVn&70ldC+;SY`k`GqNj?f{lo+b$lBuaUk_h@v_+5_cRHk) z+W6tc?xOCr)Ml?bXU`KEPsy`TI*rsQ?Mk(bhV;MZ`l)IxUvDfzw|u_HgzyH2eCJq) zQa?M%k3Jx#m)%O>a-P~@93>)n`ebdp^Ih4Ucd z+EbAlv&CJAox|sYJnfpEZW8+Tql}35`OFF5H|seGyFChMNjpE^RZPabU%vGey3 zI9gr+Wr4OKy6207riY^KA==3yorq|Jg1dL?66269FF^KDH!I&FIPup=lU6y65eq*S zfs%;OF9-BnGcCU$CieWrD~072lIAec{3@j$Q#NM`4Fz81Wgjj^p7Jw5R5)Dn5zW0} zOHUMY8Z(NuvI%~GtqZMqQ7eYXGb6#sXOHZT(AP&JE^RZ9+wneS4W{K~RjPK#{s$nsCP@4;5PVE+6 z?7H%vb&2WmzPTF5CJuY{&Dp|M?JpWSZELS&&b0O1H0%@HgAM*H=BVbB9D$hcND>BW4;zwg`F2N5t#&9h|}t(k-a_Wk7SIiJnYb#zVbFCeGvKIb>c=BKQ3wW1qRngWbZK1qdT5=sB=C z0(zjV-<>VGWMyc{{a8=-!^pexhQugcU$mZ0gAX-=?uiJ>6Cx0j`En0)KrF(~MXKc1 zhjag@b*>;1>5i$u}aW?L$34khSvhlrdf9AOt2<|Fq$2_hIV2e(x4^~o>CX=!oZ zJWb^l8hTV)tYO2P*e<%;TP9ChW2k%i%N2?(vc@+TpsmlpT%1Zpm%)h|CHx34x}K%U zeV0i-