From f99767e1d9025ef50971a0e4255741d66a0904a8 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Fri, 20 Dec 2024 21:06:41 -0800 Subject: [PATCH 01/18] add upload, download, fetch, create client --- bun.lockb | Bin 414459 -> 414946 bytes package.json | 2 +- src/SQLiteCloudClient.ts | 117 +++++++++++++++++++++++++++++++++++++++ src/drivers/constants.ts | 21 +++++++ src/drivers/fetch.ts | 40 +++++++++++++ src/drivers/pubsub.ts | 30 ++++++---- src/drivers/types.ts | 2 +- src/index.ts | 1 + test-db-2.sqlite | Bin 0 -> 8192 bytes test-db.sqlite | Bin 0 -> 8192 bytes 10 files changed, 199 insertions(+), 14 deletions(-) create mode 100644 src/SQLiteCloudClient.ts create mode 100644 src/drivers/constants.ts create mode 100644 src/drivers/fetch.ts create mode 100644 test-db-2.sqlite create mode 100644 test-db.sqlite diff --git a/bun.lockb b/bun.lockb index b73ae3f85fd1f86fedf7ea80d7d43630449e01d5..7b7ce894d39706876ff19ec7039b64489118aae8 100755 GIT binary patch delta 17082 zcmeHvd3;Uh+V)y|ua#tnh$)gSVyvkol>|Ex5kpNSG*n4V4MI{9s+5#SOhsE$cl8zo zp*2=fCDg1{(N;@)iXeuLQdJxs(1Z87_Ift?&fD|8-yh%a`>Xq$%k|vXGu_Yq3~TMR zvis-T&TrN`7eq83^V>fI68)lrCXJp^XZorM>(aXKztex?ntM~;Pi^(-!i`$5r+hSi zzFF9}mhH^RtrNS3_-2hs9X5KX<~|S-l6>*Lrd8Fn+>>B`@UZmZ!&$xvb_LieV@J@` z_Nn}op`*uU!hRQaMdUA6JWufy#VPJcXTzND3|%XSQZJagRt1~}rhR0ord0+f`{-JE z@O^LqxQDOq@gg-NecZUzu~|1&{;^Y<=0JWC+N%Vf>u0rBMb%4G<<>ZKPq{oW+RMq! z^4GOMR2Ttfhl9Xu=;?C0XLzX-$D|G&r)f=9ej9LYc2Hi|LcxCMko8}GQ}cA3hylI2zE%IHK1!Ay@{5qLkLF33#l0|rjFILJFr>sA1J^Mc7CLJBCyRp zwR&*+05s2d1Xi}XxmU>=?;MPW(ausl${Jtpi&d=had3yjtL=?PRm;8hD&D`t=CH3S zKJV5;gL8s0IL`UxYF00&!SuCLu&z}DKL?ww`i5Bcb?D5GK&xCcsT0!?I!!A^JDk;l zs=X4Yt~p_U3TFL1Uuc67WCAyW5b7z9+p~F)%O8&)!cf-;?zv!re}_Lp^;VpC>ZVf=Qc%x z1uwgw4-3w@fa09J&%q4CJ}{llc+|2tz~+1mO~L=tQniUqtau*-b6(QGEI%eZBZVF> zXlmtO2h-i}!7Mi)eY2melu@J7Ul^wiwCd;Nc1MC8w+C~akr9@A%g{01JOrB!Oh??< z(YnX1_>4;#8$KKXYt+oDHwnzRWrEr9nDH6(N6STdw&R_{<*?b07R7QSv2G)+g^~|} z1;-9$HBIxLE9se@dD23oEk`mD^zf7+>GRMK9m)Xn{22u1O6(1$!{gyF+Z&Re>6wSk z?i)3O9cx-z-7j-jsuiq%;BH*2q5k`}q*@a~{Py%)dh=4!@LJo7T*r$7I^5GV7pl{h zPxNw=F(3~u}x1f%h`0TjXS?* zoTDpKo~i|1UHUA^v+D`^$5IS(7m<(LmE)@zy9U?oUNMZLA3X|-UNOAX;PtORci+^O{y|<79F`CJ-3+Uhr?r&9E=MV>4&IXHY@Gg`+$HVf9G@da&#l%&%jsI&18W#8 ztLIB;h#9OF-aPX>tj_MR!>#>r0ZD_byD`D$I#_+(B{Sk2_mO(&Odnh}ov zV8yj_=_%r@0P#bJUC;_b+s0*cjupWmWaITHZ zu>#h^Wpf`Er^v@$&=ZXZm1)i~4pt}R+1z!`#uzY~yTf|LIdojAxVsR@?k-0-tcU*0 zh1CstVQ%_*4HnnG$I=IgvnsaE23*-XNmx}q&K4FTORDN^^yj1)IgVYp_F0v=%nV$~ zxI+==wk}5}SP#9P2}^CBkuLLhSp6l;fenI-*Tbc>9M;46kHv+KQE+(XEnA$y=<)%+pWGMVF%)Ebeye)JmzTyO;KicjQ6kRG3mHSa<1PNpf{~XyVwU zs!2+9s0mU`brn>4>gTSL=+du9a;TnTd!)9U3Du_{(8)Eh{NBLTnq^GSvi%Ge&ncT{ z5%}X8OkZuDyf(_ha>t2~2P+DB@Vuu>zgy_6sqfbvJ?eVvGg7a?VjOS=U`c%gi|32( zIWQloSGMhbu(%2A?t&XJelR!~-P7$ausFk(gEeu1qf6Eq+zS?WtHaYl2VBfq62nF3 zFTi3R+->J_9Dzl5ZSI03m&0t}J;3U8b2&P~;!?GS`8+HheAX~G!J^Yx-DvX~tY}zR zZDU-Hx(%(+p-lwj8CdKO=Sy3c`4+6l-TBxEmzlD3OhPMh*4Gp4*^PAf*|d1`Yur|} zb?3K@a|AW^7DHotvN#*)`yB-ktw^x|#>-){@a5lv2+-{`Q9K)5nupUlU0`5t;Ij#Ah0*hh6E*ga^3M}jT$jK^)n*qAx z>lx7duz0n>{=rG|D=ceGVt8g0?ps>B^9x)1A;Ch{nmGc?|9WK ztQ3digvo}8qpEwHxfQ9_o=65ndEIBcd&1(jw@&npu;{QS4(2sjP2Kr@;v5Zdhr?2y z6=IHq72(cLi*u|(ibL|atzVSnX4sL==H5VK51A>jdPuusTeRmkWiw3cB*hXie}iSs zPIL>aXB-)AT=oeqba($z@s9matvTZ?1-G=;B8E8DrT3J)DD?IfL@ta#&qk`~vKBT5 zfaTg97AM&umvCa3Qpj)^L2?DZ^cQw&XQO2k${- zOafTc42;#a-tK&?ry)r3cA=USMBzAEDtRq%zjIECLAFY*q<4~d**n1elx22QsC2Rcd$+Q8`{ZaE(+BYnzVK!>?_J9v*IWGv!NznJ2(Q&51D$jvdc2{ z7^RnGMy3^Xa%(WY(P$l2IWqg}r0g6f*kM;NN0<%nzB78Ln(H9T}-? zGRu!rHkng24$OWgfZ5L^Fv{gD%I&=|C`;y{sR6R2FZ^3Nu zGT0ydPcWCuEigZ1I(~;X`u8MQ@lREtEK~oN(#dpy@nlH-@DKAHU`C)k#eZY!flB`? zX8(U#fF4#-70K+Vva-n)Vb@i9Juo|J0A{^LV1CLn>o->UO;kP@#U;PD9-PC0MM7t} zD8V%XFl- z(#tZ-_fhrws(dob_fwn<=C)5&_HZx<{+u7&HRae-da#r2jD^lmS++}fAH9|*PA{qK zzhZVbN#+0FFw0L?<;yaq8A>n1!y#K4^n11{P?j0^SCvj?2d^ocOna`f$+YKzIqn5u z)^``4=%aW0`{MNX#mQO^e_x#b>Y{{=;<;#X2mO6=SAxO z$;D|_@i@J(c%1E=fA=FlUTUQtQ0?wa8Rk!k=lwQ#d+YZLv2}IrUf?~jL)Pcm`bxiCTYv=Q z+PdlGBsmv?-&6>DDFldPDukUBGNwWZlwB0kra=gs2BDIqPlFIL9l|jRRm3?RLJ@`R z=@5eCD20hLAcW6=5G+|UAT*i@p@f1{8qb7qn!?z7Oy;}{A!ZhY zs}yQU>?{bEDdf+BP)ABBES?P^aW;f{k~bSdmpKsbQm8Lo=RmkcVbdH44dph4b+14e z_zHx^vi=na{a%F-@G68RlKd(Jzt+^tlj1 z=0P|{AzGaCAQVx^o(G|Y9HlUEK7{c35L}WqA3~!A5K1V-O5+6(PE(k>079G;Q<%LF zLi|EoH(R`2o;#vHEitb{xe5i(X<}bTk;@eFUx(06N+~RMLr8Q(=pcD+2wmQQaF;?y z>G}qQTNF0E0im68p}!PUn7s@_{4xjwWzI4PG5HX#QWz|;`4BEs$j^t6BBc}-FNcu09Kuk^ zTMnVi3J7;8q)OKn5N=V}v;x8ixlLi+N(cj2LP(eOD61VUiTDL6zBSA;hnRkS%l8LWo%h;VOk3iCqWbGKKtg5T?qF z^$-@Xhmg1)!gN{45?wYxxJzNCv}1`|6gF*uFiUQ4fUs^Ogn=6&%#rmQA@q9(Lclu^ zUX|o`Aoy*9u$RJIacqLHlS0NO2=iqZg|v4eguM%4p`^bHA!IXzV-(!t+zg?JLiT0| zi{vPUiCZ9qZ-F3^wFN?>tq@8mGr+z>H{E#Y6Iht!XJ4wzg3oM*?73#ED4$=owf2~{Z|kwf5}Q1E!M2B8 z`Fa0HK=xg=t)bk@fDtdzqZFq zos;(Y`Y73Y(q5%;2A?nCQyoS4ZGcbwepFA5LY2lxQO_1$J!!Y=*!MS1*{j4aP(>a? z5jM`pXRj+QN>!^4ZIRNVm4-jt(UwWO(`ceVWwk^WLM#eY4J*{2W_ZaVvjdfO9?8gKC*sOT3cxJq5*J4Y3-HP5Zc+o z$YOg3Tl_wim#Nz5h`a+z8?CfX&_063=g(u5)*0y{*?Jak@X0PeW08gbYh8hIN*fP> zZs6sS_OhCq391RaO4a6zehy7kSH-hb7T%F+b)Yf4FDVUgI<&QDq6T=9(t0AjUXs|v zN2>K~m6e37(@LA7v|iAPm6oHl-q6lS!8wGVPa7YBIZb8Z1)}E>CF43>X?R_%RgoL) zF$6Qy0OrdoD;a60(q<{`8EDnj49-?se`ul7t^`fgP+702tbxc1Q`)Oa8w9P6(q2>A zU}*PbA({v}uR=FZWu+kNLgB6w`({HwF6rOchXyrNMXsX=pH-~}8q1|`>{awf<+pF_ z4fQ(`_N~2P{Bx?C+p4YgDCm7+ct>d)ptVCe2I5bpZA7}gO#K!k;)6*J`mV~_gmgGG z4*H(b-bFe>b#-59o1rz86KsOdL>UMjzH$q11^9fGfv_p}jg}O{7~&ol*=rTw0Xc8wT-7Djj)QmG}c`zryEB?RzRa=sZ6E2$J(v zjk;3%q}`AK|FVzE;nTVK0H5CRi5?#Zz6{I)_{N5(PbM%L;3LFrU#D@Cv|J3p^8f2J*}kU@@=+SPCoy zmIEt*l>iSv9(Fw3-UikHYk~E^2WWdYu-DKe?!MiTJ{}!^0PvwKp94Pw^aln4g8&|b z&jLL9h5$o>VE|tr@XY%a;5qjPz*h@A-+l(J0>1!!LbMYdK2x~qKCTS<&l0a2p*egl zvKQcyw;$j!cM$j(C<1tn@!Sdkssh!3Ex;7GnhWs!nh(4V@W848gaP${M}P#N1F#JF z%Yg!5J@6rb)2X`l2@)p(9!ori&HxvH%W{MMjlmFK1SSA{+&vCR2l$Abk8-L2Re>O& z8W0TdYzYN;knrHB3Dg2=13WbNzJf1H?gICK`v4!1@zlUOM{NL*07Uy}g|FI-5}Wfn ztRe)j4_@4yMycZfUm(3NQ%$2vIM0$t0lvl?g*}o19u4#c`T@znGeCb}s2ntn;2gd; z<7>*!;5FFL>wr73*P{A(_{URc6p#to`UZ>_#%WaUHIarp&auy zraSoRBMFF-WIrPzoKNHz0(??G6Zi=M8VlgBGWcf!`~w5+4K%z5!Mi4h{futOd*Hzi zNQD3|p#A_~$cBP>d0WCL07bx0z`MYCG|9IaSpZ*({E2dWPj(yN6^B=t3zF_I*5;gn z^fj;*J@Q@OI0W`ZU@Fi6c4Odape~xM4`w-HBf9AXdo9Yf0>=P{;r$&j!Q;I>&;p>% zGo5F6Q-Fsv4`Up@Htk~JLVshhu74>v${7v&M6)c=5^w=gKsc}o{rG^7ppQ3!r%}Hb z;O&ph;T|7;7g3vTi?s<{5O;x1g-@>4)9)-`P3RH9e;AA4djd10T5WI z2AG|=fD@|WeJIplRd@=_N@2(!1{^_|w}2kdvtYLdcsu$yP!sUB#rvRS+cU^kQ#j~VrHxm|S&geK6~ z7YF?)(hTM~paaspJN^*qNU(R@y!EG#&A=GkSCo(ymZe&$adlqGQ0C?B2cMT7Q z#?4$d0Gxr^GBp}kTNYzn>c9t14r7xJn_ZXf78CSfTXWm6TkqE4f=B~~0V9B+z+|=s za9eYn+^)Rj4FU5YV|&=CIi7Vp93m^Tp;RD>odGz+JsQsf2E6PFWPS$9jsY0JOfXj} zzlGjGhujOFBYg}g1jYd`07IdB9iPX;i(itmfs3fjMp-Z%JQ+L*cnNqBxP*LuKb;6n z0J2p274R%z3NRY==747_?PYL1X#6_J(dU430e)qiN)7#MGm)4<1L;@6oxyX#+kv&f z8sIHpHLwa;3DAiZz!D%2SPZO3y+z;!zE0oJ9HUI!V$jj%TWj5eIi@q~LFg!KT=rgy;a z0-Jy>04uTZ`$%(oc7SnV*FFP(3VZ?_NB%zWQJ@Ie3w#6|0CoX80X|aL4`w;$e*kpi zgzZ6MH^Ao$4E<3wMvZ((@j>v1z!BhMfE^wN)1fcGbn+`Or|?T~1AsfbI@ljL4V(f_ zitaM1JbD(%5`cTT8JLGrcT~NEbRal4#=2d}43;)7W1}8b_>;@1q3bOQePfNs3gqG` zV_9g*eO!rA)*0__Yw4TiZ_@`gi-?K9pN~$Fq%-()^La876zct%UB5|xd|xGJ(;k$I zjA+Jk@5+iZMl^mx@)fB_e9wYTO59mgz9@ZQh5m|89Oz``fTCxocK83Os@^6(G9oIX zMYFrI`mC`OD8+0BcR~n&4ZtT_3JV3weWwwei*(+#YVW`Y>}j#Hxls(bTjEY zxqIFiQT;zRBn#FXLB5fZ5iPX{QatL-CazCjP+ zK2qUZ+!w}4!*3Bu)lQ+M{{b@wY$KM&v_51SY zcj&o?1bzRY%Mh9TJ+kJ?*6)o{Jhoi=1LoBGW4(YG>o&Pug&_#2+-$?oq z&3c+}%8Vb4=3ZB2-@lCN^78?s`u`q}JQ;is<9O|N)X2OH&)3MD%f?b$Q-7I!#R&0+ zL~kRTZ?Rmd(L*03XG;-@5fbQ@tMJRo$##j8a>Z3NrSZk!zXJ(%?FFg!hM$YsUng{k=lA{&1yV zlalL(vxWCZf%Z~Y;#*p277Db$R>0RW?~gC@H{beIo;`a2IW4VCVpWw-?i<0<|AvvI zTLPYe9l3$QY?H_S33t3xe@~)s8Y66hm1WsYtd;@t#Z3(5MM?Sjek5?nj!cV z$_1EK_?-G=sdCGh8tVO>W>Uk5dfS)ucRpM=o@pE5t<^~)Z)2x=f3_L6@QKW!E8P=p z`lC3W`0@U*v;6BFpQ<$DbmxaTo#Ys*s*NH&?ikg*(UFq}G4f@1j5KQlM@!6~$hEds zh`vWM{)EH-eFdvcG8OM#p8pq0cy_<@L91ND|NHVSkpRZcv#Xu|J&M((8J8k9=v`y# z|8PS6=VQY&yCZ6Ne{lM*A=;hx_k-U+gqz{O#MgfBIR9;c$FCwNL+*R`xKp;=$DH1g z`}YlJ%fDE$YF&DN0J~#Sc4@7m^2gCnw6$XGp^JK$A-2%aMQLWPI#Z-qM?|_wsMOTW zkSc$1Zc10(?1Oh)PlGflZDnKrY>gH2I*%4UY?q23=yhFrcE(QW?t zkKr5+ab9z#OQ2yc)z8RA!~DYLtSysG^Kz*7H?$+Zd~C;eGb@dMIGAao`|=JS z_h}vLCz;+~;GPbcw$lCT&@WLsG9r>8wo2=lCDhj(funW0uh|5Dhp^e#T&=H^-hO7H zzDx@I%;-?>4`1&$f4)bTjXzaH8_m%MJlrSO{15~0FJ|8wHaL4#_tCvkfM*2?T#$MW zG;~>>cbHE7mb?n8;r)GXUZa{{?XLBHgvY055%_su)%tS4VMf*P{=)ZoeDbN01IE3H z5>aRn_pVX(rGmfN9d9%H`@`+q@}a-E-PW{`yj0HY-u%HYd^flGVAxZwd7)|+(J}(} z$m%!0xwixAZVzg^^14mmgJC>Mq|S zPt8IIuMEKk4#R2f*(%39Dd^2=)GUc;KKX%7q`ve#H|4ObqV9Wkm(V z-23BU^Heu|+^QD2D4>??_j0uY{JAc^f#wKXWQ2?fG^6xGvN{meyuY|jyFEGANZ5{} z#uGTq#2~p5h-w*9t|Eq*D+v{0&6DJch~jcNNO7I~RuNu`Mqveyz82bwn zTm{uOOE=Oo$pM9We+B%5*#fc=mpfd{f127PlZ)`7Q+wnlzl3>!c9$ldVeFUvO#eze|!YY_M! zCHSoL!l6kc+@BcghViN4PYv*&@&2KJ`_d}d>=B&iw7h>p{YNGITaS-vQ#(?gbQvLq qg~8_I_he@qGgOXO^zoILNJncAecW@auW zWTaFIq~?|>ezR=H3YRhy%gV~sO#QvDIrpefzn<@(&+GZ4sgIxg`rO-fU-xom&f$EM z8*%mJh^4Vj4qPq1dhf17J-d7#w|VWED><)j8o2VVD%V3^`>S_qKxAq2(R~99`J7rd zDB4l_S^m_n;n@>o#*BS9)yP3EZx}PP#`sDYxi<`>8unUP^oKJ>jgp*Y*p;w{Pe}6> zoUikTj~qKes!qYKOn#c?L7F>jz5|XT|4e7Yh=9FaBEqxtEYqlf(Z*#OA#fZl_QBPL zQ5CKmU>X(SLvRorR?hS}m719`X;SKhteral$QOp;CVv_2Re=Wv+U>;H`>&x<5EXXGK`Bu63Bd1;e z*%gMb$%}j>{FHO0TsekjcjA)VG?0vOiayuEOmlm><_jJu0fX$ z3g0z+5qP|^=?m|?RfZ9b9telS4d_TBR8qz6_#iA1=!?TL9d~JNWsi7fyAZp)MMSD* zQ=rTrqH4Cs6|iLrc4JEUt5g9qww+xJvL_pTbJ=N`w*TFKN;4ssu z4kutsczlw}8bG`1Gy&T^}d+)1R%iQVB zs_DtjuVcrc8!WzVgk>2GA2(s_h~b8jj4fSf)U#tWX6!iM5QdK!F(LIa!`Kk5yP7y5 zLxy&>zP87X96b?xUjy64jB%1{81t}Y$Ujlew=5^5PMVNKM(TtKF(cWfrMiI+VOdst zHK&dtH)Bj%o5pqn&%&}AE;h2u8f)GIG$xRUOFO^sz%F1*y}Mi3x(AkC#!b$Y&W*0*OFJXe z$4(i;CQ7r*XU{B-vkPvAWsQuAw+l`fDb)g~GoZ3o`8B2bR%>3n?s_fguhwhIeN(b+jYb6kW=1!p@+x$;SEDIHO&-v1;aF&Av?VnI(gh4 ztlqZOB~g``W;^$Oq!vh)_d-{X`!}q+u}r(TX&SMR|6>M?t(GP2Djs)N5~sQp5eXF&f~7n5zy1Nu6x`Qu*C5w z@Av|b`yy7HZH@N02V16*bki!tYIoD>#la)3*dtqmbt^BH)2at~hBtq)$2}dZ!_9i1 zVcmsgdE<%`1GzxRm>4gkABrX2;imGqS75dG6}-^b#oA3;vQ7o#lZ|L zal)=&fR%DH?{}!Wt&C#-ZV(;L^t;~LK;XOQ>&nl@%d^=h!$E|5#gz2w>x zosMPKlvcN3-5it|pmHlRY2CP_^rb?8Z!Mh1l4t~Y@!uU2Y+D3)u*b|)xj|;Oy97}x zFe4p2<{c`d65`Wbh`JyW)3zShr&tepW5y-BT5{p(D=GIpQa9t|`W}mTbWL`}RK;QM z`GLuV}o9@?2({KvFXO_8fhJW!I4ztIu`mZrjPW z9(Ov{tz~lvOXi3%_VTzKq0(MvMcO0P3Eklv!;@Goy)k{0-G@oZ<|3rsJ?=lSZhGS$ z$aSn6dA6s8Sh4|pmf1jMRAcW<8CG@ky{guV#BUrLU z33hvrJDBU*Eth*@>D@!g-B|bgx{-P{xPsjZLmJl2FuNUG^&|{#-voA28DVt$Jfgj& z;)W*%VkA<+k_mU!uE~V^rsW}0gS|0blHG-*22vtG)|UG}=36Q^(#&?8k5+foG@l^WxiwjaLu=b*T)ySI981nB--Nn9 z!4h8`zPxf=ti=lOjz5?fh!Ky0oW6KXz>>vANj6jgmdvs3-}hKj&pw7Dx$Mc#b<6T|SLEU^B{_!o zBnDzg7JIaV$GsFwyk#JLJnj#$?DH$G8`IO!UaR&X@5GXGk9wSYk70?^4xeL%SP57x zw{af#4_Fdh>QS;DUlgQ2j+piyS0>io-aK|fzNBoODR8Qc24*{Fr^crDhx8QJi;ZRN zmC?}L?S7<*KM)eljabry%R9a#(ZcBBTWg*9rXib)Xt9FkVD-kb z)CHHB7>Ff%AlQ51qr||b_Fnac&b1D!tt?yj2~rY#-{I(P#P<$8uLM6GOZGPh7Xf|` zOQPXh__D_CFn7B;@fF0r&dnjEXBbc3!@4z7fqXHMy=l*WXDk^lSDG&qEet&A(?-gO zBjtP}nZY*$nIv`%N60QLd%eij`YM)864Sy2_U22Bl(g5&QY@br>OwWs^7KKw5 zlj%@RVmV(r@hygfs(Z3)8mTtEIDSsb_FjS=&i5DD@b;lT2}}I-g~3&b)y$hWAld!1 zF6CPuu9O766nOK}lij09$u9EwZsw}o7Hr0E5Pfl0aSk)lYDovG&k;w0_hj2Z_f$;# zAZ7kt?_k-J^A{=UnbV@J$JxG>=^g!OihDsT|CGrNIf*3;k=;|kcb9njdVz08vM7Rm zTj&^;gbVjssi7WwhTSSI()74GW42M}IcwcZNy_!0ywq}h;!%&Z}}c>mgJ6rY(swR86W8k;cD35 zX3LjWLU~Q22(%Vm`*}Eo}n|IfTiGUSU$p%T*yCB z@GG#C&w-`k&2R+#uC@>A{9j{fq3QI)3fy=``!m?a`h2LI|+ya)*?JWJrql-VSbpGvJ6}_{r-$ip*T~1iKzZaHv zx(CX-uoE{~3iQwwZfB{`OIPfp^M$2+U(Nks+4jS<{SYiO`Yj6HEEYI{*L5s>uK|(ieosC@Q6&&{h74rOVkm|8^Ev z=W1P8x}K+PVX^0HTUhJ`unhgFK(@U!usE=whR3}3-%HiMmnwU4|9h$W_fqB7r^LUP zs(&w4|36%+0_;oG{G`cdLDFQ$*X32_Oh-4pIgD~|Ogi;Z>SGfeC@e+i^ zOAuPAVMDd%5&Een&m%N`9%1qG2m{nH5lTczS&cAAEnJPTU^T*d5r(Lw7Z4I(Kv?qv!u_gD zgtH>_Sc5QBty+Vyat*>Y5r(O*FCuh#5h3qIgyHIn2$w||x)x!i%3X_)vlb!fC4^Kp zo$%LtRzF%e2`3RBd=R}dDw;xCbxP_##<2YGfq1!02LFH~k$k~Pvv>oAPHDo)&;Oz*zMA)R<1qgu!2$=;4xvEHnLJ^`05jLxg zLWJ}}gd-y4sR-#dya-`N5yBQVOD6k}2r+LXu$7#yb!e??Bk5 zj)_nrLds5r0<~}_!h)R$=S3(|NxKjdcOk6Vg|I`FiEvhg9`7LRQmfuUSosdZH4%2J zuDcPs>_*7jjj&f;5#h23L-!!;SGju-a`qqu?L|1ChU`Tcycc1Y2nUsWA41?hgv@;i z#i~ezLJ^|&Bm75Y>_EBgvRe7EPfB+Gj&Xa5)o1kB7C719zEp6J~Ett_UaYa*ObT@NF4IqV23$UE%# zwvB54lfx>=Dd+5OI?g%O^+0D0=Q*cYu=b2&l|yYUb+jqy{fFagQ`J81NEh33UE{%4 zRk=HSmHVfbY7^{y%Y2~VLa_5U)A?FGU)2Fl$uheabO?1`H62G=6|@a=j^16q_UHCM zoBgFuf9_0C^}ldFZH`j;UpPaWq{;0cJ~b#JKW*iP+$nuiCsJ#2r|7CU`<>EL3JGO;ppxvvd|uE{#Yu3GXzm@vOpFM-w&S z>00YWdV}gCO}wL9pP{q*koB?Fp3qufv`@5_t+jq=pQ?4GgkNq3*2kQ!vv{!QyMrik zeNt;Y$Tfo01?jOWGt&@rp3WLVIz(&pwRRs`H9dn1w8lePBUE)bP7~F2)>ArbC|O}z zTcouI(IU0>wAO~9T~^CzBD6$@?irmmoUCI7MaP|6Ec39+_|`cxw2m(F3q|C%(FRaY zo%+@pV%Ap|zjZb;e^*hbosClD-k5x@=(ci6`)}y|skK+pk|`&FxT>|yq}!-jrx}sl zHE_bqzSoIDSj zNg$kPvc}#7Wb^PH|+OE^&#pqhS0w z=gz9`zi^7rr`6Q)P^+#Qe9bvETker91ac2e?yJc?xtU-VkY_1!@T7wbAouU4fT+(3L1%mH%KUyegL{^Yn@2A&1a zf#qNYSP52v=YgDia?Z)Qwidhu)`1P+ZQ9-ec3H;qWXqi~mVVy?at~B)(Dnxdz#uRf z$N_jCkVEeQFcdrpkzRU zz}@kWDRmgggPqsatN<${Mh=nsKpw9>%H~LeGe8f}7xV-D!2mE+?G3QPvgLV}JU;9I zzrs$=0l#B!qPpBgm*Zv>NC$HJUXGg9;7-s1+yy!TdB%_c}3n1-k-yK+zP)1C=`JNTBtkTb@aD z1I^SBx799&yW++oAa~2N!8rmn0q}Pc^1xmm`Ww&C@J@nPrVhBRZbP=?!B(VvAXiQc z$b-^QSgvc&k(O)OK5!c3fa5eN&kiO6d026ga`OD@Hz1c9xwITp8Re}_*`FeP2=eJs zp21Bbu#bURpf+|r&>PgH$!J*030|g~_Sl;!*Ak8g2k`!PSRm(m8_*nxEr+=r-VK4A z%5n;G>N<=s3zk)|hMDI3>Ow`U(SR5!3tE6U&2OP)6Wc1t3bq~yaECGuJ}o_RHaEQU1V{C8PzvPx?=esUJ_VnE2yhg9 z2*!hO5C|?g3ffk;J_uKhn_DeX%A?%SpITf?+}L5@Bt4w~@~vkyc!j*0U=P|O*cCyB zTGreOOOlQ*k~I{LhK~aI5-P2V)HC3{iBzYk?!HfF_fuvID=f4JW)`_^ zKr7N6L6nMV!D&&Gr1&VKXawevmd52PsdOxP(n($HjZ8o{SelaWrT2ozNlSB$fs7>@ zt^v#Us#>IFm&yY1J1p(hL)UHA^;N2kPzT7`uMdkO()n8KhN`TERWG{>QfKf_zI8%x z2oBPZ_$a=KU(33-QYariZJAnP9TfqKt$B`Bv{XhY>@ zurw}>H6@)0%NKDe>@Opl*s;2f*wXml>x<1F6B+TUEY9lKN50ofwS7R=hJP$F2I)?| za28Tr#?TUN1L=Bb5-Jb2#I_-*2c&K*H6z}N?$DO3J3%r?(rF3RE+Anj29lowF-EW?Nn(zD@CGu4-ij0UQpuI^flPOUyrwvb?0Bo@#Ldt3zgO5ReG=2hxuCA->5F2Z#nF zeFW?WQhp$iw1hx-2=-tgx?dAreDu$Tv?=3~K4jaWXB&|uBrubKtlo#f-wL=aiiXi8&XjU)I7rKUxynt=O*vsVh$U9h3;Z zik%B2+Tx@{J9`uVh`4Z)3=>eF=RrjUoDYujW1QKJXqm0HnirVR7gKSe*PA zmMQ!QZUAIMN5F3I1^66%2Bfa6uSTFb_|n;!>joKe);5QIrx_V1Ne9E}9{U@`+o5V; zqP5wKD{v%PHBGZ+LG+zg^L6|!`ww4P5t06P)9(2u<-pDtzJ1$aJ{T9K5kTM67~o-eda?GRe4ex?>TSKXypyHrXk)_ZDD zskIvMw^D0XByS!0-WWNsw8D~h1J72X#g?&gv25S9YRPeHSou8^sKQeJ$f+(Lx8fZq zovPUhtGn4yT|Hq%sHgT@HPmY-td^18Ec@M-7uQ#PY|E~U>ecv?VZSqG7saVrp{`KX z`NqZ0s^=-Y=h>&M=Xur^^$pdktHswD zP6HME9Tcb5eq(Jj+o%EGT0I|@_{OT?D08W2PSfxo zE;Xji3O6nF)oJ3STdb~vE;GQxC#@R%CGpVjh}qvpP-Dej@^-4uXRI{yklJzvKf+Y; z8QX#I?}^et6c(!&U;Pddk57ti9h+d>*1X@_NdJ3t z!QU=#ShvNX(pxj`Yr2PazWd+8F+%-ZMvTU)lylZ<$3?elao!5ojmDehRHGkoxsh7( z1LJL>j$N`6R8NPihIyB&^5e}M_0k`N>iJU)GFqMbkzv^l#ot~!X2s@}f1%Y^Q$ygITKeht<>cMJ__+{D8|ceq7!02T`)i{I4aT zj-Itzs&SXt0RA@;_pko^qj^Pnugb*Q?rm4QFR?c4K+~WsEW4wGG)u+zMv!4c0sR^ak3`XD>Y_=zLVo;nvBcEw!1jP`G2iK$1-UI(+ReREu*h;nnUVh22gA=^ zkL|W(@_W=~Vr7GMSNmPA)jZi77U24jms3@Xa;~$H{`XGPK5oAK^xP_wDX+)fO(g`n zhPj`sW50j6QLPRn%KrCPmu$*zIVXPlS<1(8v7!BZ^;0mC(xn3h0fN<9l}5e6BK;epXq9!(kY#pnkn zt)PSilXkDxSNkfsx|{P0bNZF5{~q+)|m+olHH ztCJYs(uw$JkP5KRl)C?YEcy?_U3!2m*#%_fcvj$I#S%d zYOQutF4F%->^tTA46WO=`85gl%kJO0{SrCrO7 zX})p!R?X>n^+Yw_Z2gBO%&F>zP^wA@#b5vXv6p6*KiaNxhoufPu9dwLeCNSEky*S9 zQh|WxaD0}B*EX2htm)j;5tYq3>QtyJnLi4Ms?J8+t=3j|eONxQI=9LAEC^Fm!(8os z-v(7-m}_Zv$LrikrGe$!qqbI`zqLV6ZD$p2Z`|koUrL5x%gpt>YtRQ1_9ZPz*ZGO= zr1k4J-xE7g=Wi{Y64K%B$l}@9l_>vc-KEu^*gJatQtYbOYW|*=59P;wupXONBWAw( z@lP-24ZEY>oYr bt_WAg^@4B00%|6!H;Y`g3u^TXc-QGG diff --git a/package.json b/package.json index d398d99..d84b0c0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.354", + "version": "1.0.355", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/SQLiteCloudClient.ts b/src/SQLiteCloudClient.ts new file mode 100644 index 0000000..10b814e --- /dev/null +++ b/src/SQLiteCloudClient.ts @@ -0,0 +1,117 @@ +import { Database } from './drivers/database'; +import { SQLiteCloudError } from './drivers/types'; +import { Fetch, fetchWithAuth } from './drivers/fetch'; +import { DEFAULT_HEADERS, DEFAULT_WEBLITE_VERSION, WEBLITE_PORT } from './drivers/constants'; + +interface SQLiteCloudClientConfig { + connectionString: string; + fetch?: Fetch; +} + +const parseConnectionString = (connectionString: string) => { + const url = new URL(connectionString); + return { + host: url.hostname, + port: url.port, + database: url.pathname.slice(1), + apiKey: url.searchParams.get('apikey'), + } +} + +type UploadOptions = { + replace?: boolean; +} + +class SQLiteCloudClient { + // TODO: Add support for custom fetch + private fetch: Fetch + private connectionString: string + private webliteUrl: string + public db: Database + + constructor(config: SQLiteCloudClientConfig | string) { + let connectionString: string; + if (typeof config === 'string') { + connectionString = config; + } else { + connectionString = config.connectionString; + } + // TODO: validate connection string + this.connectionString = connectionString; + this.db = new Database(this.connectionString); + + const { + host, + apiKey, + } = parseConnectionString(this.connectionString); + + if (!apiKey) { + throw new SQLiteCloudError('apiKey is required'); + } + + this.fetch = fetchWithAuth(this.connectionString); + this.webliteUrl = `https://${host}:${WEBLITE_PORT}/${DEFAULT_WEBLITE_VERSION}/weblite`; + } + + async upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions = {}) { + const url = `${this.webliteUrl}/${databaseName}`; + let body; + if (file instanceof File) { + body = file; + } else if (file instanceof Buffer) { + body = file; + } else if (file instanceof Blob) { + body = file; + } else { + // string + body = new Blob([file]); + } + + const headers = { + 'Content-Type': 'application/octet-stream', + 'X-Client-Info': DEFAULT_HEADERS['X-Client-Info'], + } + + const method = opts.replace ? 'PATCH' : 'POST'; + + const response = await this.fetch(url, { method, body, headers }) + + if (!response.ok) { + throw new SQLiteCloudError(`Failed to upload database: ${response.statusText}`); + } + + return response; + } + + async download(databaseName: string) { + const url = `${this.webliteUrl}/${databaseName}`; + const response = await this.fetch(url, { method: 'GET' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`); + } + const isNode = typeof window === 'undefined'; + return isNode ? await response.arrayBuffer() : await response.blob(); + } + + async delete(databaseName: string) { + const url = `${this.webliteUrl}/${databaseName}`; + const response = await this.fetch(url, { method: 'DELETE' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to delete database: ${response.statusText}`); + } + return response; + } + + async listDatabases() { + const url = `${this.webliteUrl}/databases`; + const response = await this.fetch(url, { method: 'GET' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`); + } + return await response.json(); + } +} + +export function createClient(config: SQLiteCloudClientConfig | string) { + return new SQLiteCloudClient(config); +} diff --git a/src/drivers/constants.ts b/src/drivers/constants.ts new file mode 100644 index 0000000..ba82d3b --- /dev/null +++ b/src/drivers/constants.ts @@ -0,0 +1,21 @@ + +const version = '0.0.1' +let JS_ENV = '' +// @ts-ignore +if (typeof Deno !== 'undefined') { + JS_ENV = 'deno' +} else if (typeof document !== 'undefined') { + JS_ENV = 'web' +} else if (typeof navigator !== 'undefined' && navigator.product === 'ReactNative') { + JS_ENV = 'react-native' +} else { + JS_ENV = 'node' +} + +export const DEFAULT_HEADERS = { 'X-Client-Info': `sqlitecloud-js-${JS_ENV}/${version}` } +export const DEFAULT_GLOBAL_OPTIONS = { + headers: DEFAULT_HEADERS, +} + +export const DEFAULT_WEBLITE_VERSION = 'v2' +export const WEBLITE_PORT = 8090 \ No newline at end of file diff --git a/src/drivers/fetch.ts b/src/drivers/fetch.ts new file mode 100644 index 0000000..f168ad9 --- /dev/null +++ b/src/drivers/fetch.ts @@ -0,0 +1,40 @@ + +import nodeFetch, { Headers as NodeFetchHeaders } from 'node-fetch' + +export type Fetch = typeof fetch + +export const resolveFetch = (customFetch?: Fetch): Fetch => { + let _fetch: Fetch + if (customFetch) { + _fetch = customFetch + } else if (typeof fetch !== 'undefined') { + _fetch = nodeFetch as unknown as Fetch + } else { + _fetch = fetch + } + return (...args: Parameters) => _fetch(...args) +} + +export const resolveHeadersConstructor = () => { + if (typeof Headers === 'undefined') { + return NodeFetchHeaders + } + + return Headers +} + + +export const fetchWithAuth = (authorization: string, customFetch?: Fetch): Fetch => { + const fetch = resolveFetch(customFetch); + const HeadersConstructor = resolveHeadersConstructor(); + + + return async (input, init) => { + const headers = new HeadersConstructor(init?.headers); + if (!headers.has('Authorization')) { + headers.set('Authorization', `Bearer ${authorization}`) + } + // @ts-ignore + return fetch(input, { ...init, headers }) + } +} diff --git a/src/drivers/pubsub.ts b/src/drivers/pubsub.ts index bc65c8c..bd1ae92 100644 --- a/src/drivers/pubsub.ts +++ b/src/drivers/pubsub.ts @@ -2,11 +2,12 @@ import { SQLiteCloudConnection } from './connection' import SQLiteCloudTlsConnection from './connection-tls' import { PubSubCallback } from './types' -export enum PUBSUB_ENTITY_TYPE { - TABLE = 'TABLE', - CHANNEL = 'CHANNEL' +interface PubSubOptions { + tableName: string + dbName?: string } + /** * Pub/Sub class to receive changes on database tables or to send messages to channels. */ @@ -20,26 +21,31 @@ export class PubSub { private connectionPubSub: SQLiteCloudConnection /** - * Listen for a table or channel and start to receive messages to the provided callback. - * @param entityType One of TABLE or CHANNEL' - * @param entityName Name of the table or the channel + * Listen to a channel and start to receive messages to the provided callback. + * @param options Options for the listen operation * @param callback Callback to be called when a message is received - * @param data Extra data to be passed to the callback */ - public async listen(entityType: PUBSUB_ENTITY_TYPE, entityName: string, callback: PubSubCallback, data?: any): Promise { - const entity = entityType === 'TABLE' ? 'TABLE ' : '' - const authCommand: string = await this.connection.sql(`LISTEN ${entity}${entityName};`) + public async listen(options: PubSubOptions, callback: PubSubCallback): Promise { + if (options.dbName) { + try { + await this.connection.sql(`USE DATABASE ${options.dbName};`) + } catch (error) { + console.error(error) + } + } + + const authCommand: string = await this.connection.sql(`LISTEN TABLE ${options.tableName};`) return new Promise((resolve, reject) => { this.connectionPubSub.sendCommands(authCommand, (error, results) => { if (error) { - callback.call(this, error, null, data) + callback.call(this, error, null) reject(error) } else { // skip results from pubSub auth command if (results !== 'OK') { - callback.call(this, null, results, data) + callback.call(this, null, results) } resolve(results) } diff --git a/src/drivers/types.ts b/src/drivers/types.ts index 6a1c7a3..c92004c 100644 --- a/src/drivers/types.ts +++ b/src/drivers/types.ts @@ -135,7 +135,7 @@ export type ResultsCallback = (error: Error | null, results?: T) => voi export type RowsCallback> = (error: Error | null, rows?: T[]) => void export type RowCallback> = (error: Error | null, row?: T) => void export type RowCountCallback = (error: Error | null, rowCount?: number) => void -export type PubSubCallback = (error: Error | null, results?: T, extraData?: T) => void +export type PubSubCallback = (error: Error | null, results?: T) => void /** * Certain responses include arrays with various types of metadata. diff --git a/src/index.ts b/src/index.ts index 3eb0db0..386cbc3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,3 +20,4 @@ export { export { SQLiteCloudRowset, SQLiteCloudRow } from './drivers/rowset' export { parseconnectionstring, validateConfiguration, getInitializationCommands, sanitizeSQLiteIdentifier } from './drivers/utilities' export * as protocol from './drivers/protocol' +export { createClient } from './SQLiteCloudClient' \ No newline at end of file diff --git a/test-db-2.sqlite b/test-db-2.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..7f55b6856c1d46c636a8e0d7f52ff87d29c31e6f GIT binary patch literal 8192 zcmeI#u?oU45C-5&TimoRrR$9fB8V?wmEzJx?ckVHD^#r&8ytKwU#pK`N~xJ>)tn0!hMrD2nav` z0uX=z1Rwwb2tWV=5P$##x(lpD&-Xl0YL#45otC;zYqbr7IOI53Mj^}kN%BZ$N6B%x zk0&fI-|E>^VF#Q|gltG-D=lk_R$BNKT?{ zGGclDboZtkHh=S)y!!XYF$n<(KmY;|fB*y_009U<00Izzz<&$$gzNdi Date: Fri, 20 Dec 2024 21:06:58 -0800 Subject: [PATCH 02/18] remove dbs --- test-db-2.sqlite | Bin 8192 -> 0 bytes test-db.sqlite | Bin 8192 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test-db-2.sqlite delete mode 100644 test-db.sqlite diff --git a/test-db-2.sqlite b/test-db-2.sqlite deleted file mode 100644 index 7f55b6856c1d46c636a8e0d7f52ff87d29c31e6f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeI#u?oU45C-5&TimoRrR$9fB8V?wmEzJx?ckVHD^#r&8ytKwU#pK`N~xJ>)tn0!hMrD2nav` z0uX=z1Rwwb2tWV=5P$##x(lpD&-Xl0YL#45otC;zYqbr7IOI53Mj^}kN%BZ$N6B%x zk0&fI-|E>^VF#Q|gltG-D=lk_R$BNKT?{ zGGclDboZtkHh=S)y!!XYF$n<(KmY;|fB*y_009U<00Izzz<&$$gzNdi Date: Sun, 22 Dec 2024 23:33:05 -0800 Subject: [PATCH 03/18] formatting --- package.json | 2 +- src/SQLiteCloudClient.ts | 111 +++++++++++++------------ src/drivers/constants.ts | 5 +- src/drivers/database.ts | 21 ----- src/drivers/fetch.ts | 9 +- src/drivers/pubsub-refactor.ts | 147 +++++++++++++++++++++++++++++++++ src/drivers/pubsub.ts | 31 +++---- src/drivers/types.ts | 7 +- 8 files changed, 232 insertions(+), 101 deletions(-) create mode 100644 src/drivers/pubsub-refactor.ts diff --git a/package.json b/package.json index d84b0c0..ffea1e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.355", + "version": "1.0.357", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/SQLiteCloudClient.ts b/src/SQLiteCloudClient.ts index 10b814e..aa0a067 100644 --- a/src/SQLiteCloudClient.ts +++ b/src/SQLiteCloudClient.ts @@ -1,117 +1,126 @@ -import { Database } from './drivers/database'; -import { SQLiteCloudError } from './drivers/types'; -import { Fetch, fetchWithAuth } from './drivers/fetch'; -import { DEFAULT_HEADERS, DEFAULT_WEBLITE_VERSION, WEBLITE_PORT } from './drivers/constants'; +import { Database } from './drivers/database' +import { SQLiteCloudError, UploadOptions } from './drivers/types' +import { Fetch, fetchWithAuth } from './drivers/fetch' +import { DEFAULT_HEADERS, DEFAULT_WEBLITE_VERSION, WEBLITE_PORT } from './drivers/constants' +import { PubSub } from './drivers/pubsub-refactor' interface SQLiteCloudClientConfig { - connectionString: string; - fetch?: Fetch; + connectionString: string + fetch?: Fetch +} + +interface ISQLiteCloudClient { + pubSub: PubSub + db: Database + upload(databaseName: string, file: File | Buffer | Blob | string, opts?: UploadOptions): Promise + download(databaseName: string): Promise + delete(databaseName: string): Promise + listDatabases(): Promise } const parseConnectionString = (connectionString: string) => { - const url = new URL(connectionString); + const url = new URL(connectionString) return { host: url.hostname, port: url.port, database: url.pathname.slice(1), - apiKey: url.searchParams.get('apikey'), + apiKey: url.searchParams.get('apikey') } } -type UploadOptions = { - replace?: boolean; -} - -class SQLiteCloudClient { +export class SQLiteCloudClient implements ISQLiteCloudClient { // TODO: Add support for custom fetch private fetch: Fetch private connectionString: string private webliteUrl: string - public db: Database + private _db: Database + private _pubSub: PubSub constructor(config: SQLiteCloudClientConfig | string) { - let connectionString: string; + let connectionString: string if (typeof config === 'string') { - connectionString = config; + connectionString = config } else { - connectionString = config.connectionString; + connectionString = config.connectionString } - // TODO: validate connection string - this.connectionString = connectionString; - this.db = new Database(this.connectionString); - const { - host, - apiKey, - } = parseConnectionString(this.connectionString); + this.connectionString = connectionString + this._db = new Database(this.connectionString) + this._pubSub = new PubSub(this.db) + this.fetch = fetchWithAuth(this.connectionString) - if (!apiKey) { - throw new SQLiteCloudError('apiKey is required'); - } + const { host } = parseConnectionString(this.connectionString) + + this.webliteUrl = `https://${host}:${WEBLITE_PORT}/${DEFAULT_WEBLITE_VERSION}/weblite` + } + + get pubSub() { + return this._pubSub + } - this.fetch = fetchWithAuth(this.connectionString); - this.webliteUrl = `https://${host}:${WEBLITE_PORT}/${DEFAULT_WEBLITE_VERSION}/weblite`; + get db() { + return this._db } async upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions = {}) { - const url = `${this.webliteUrl}/${databaseName}`; - let body; + const url = `${this.webliteUrl}/${databaseName}` + let body if (file instanceof File) { - body = file; + body = file } else if (file instanceof Buffer) { - body = file; + body = file } else if (file instanceof Blob) { - body = file; + body = file } else { // string - body = new Blob([file]); + body = new Blob([file]) } const headers = { 'Content-Type': 'application/octet-stream', - 'X-Client-Info': DEFAULT_HEADERS['X-Client-Info'], + 'X-Client-Info': DEFAULT_HEADERS['X-Client-Info'] } - const method = opts.replace ? 'PATCH' : 'POST'; + const method = opts.replace ? 'PATCH' : 'POST' const response = await this.fetch(url, { method, body, headers }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to upload database: ${response.statusText}`); + throw new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) } - return response; + return response } async download(databaseName: string) { - const url = `${this.webliteUrl}/${databaseName}`; + const url = `${this.webliteUrl}/${databaseName}` const response = await this.fetch(url, { method: 'GET' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`); + throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`) } - const isNode = typeof window === 'undefined'; - return isNode ? await response.arrayBuffer() : await response.blob(); + const isNode = typeof window === 'undefined' + return isNode ? await response.arrayBuffer() : await response.blob() } async delete(databaseName: string) { - const url = `${this.webliteUrl}/${databaseName}`; + const url = `${this.webliteUrl}/${databaseName}` const response = await this.fetch(url, { method: 'DELETE' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to delete database: ${response.statusText}`); + throw new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) } - return response; + return response } async listDatabases() { - const url = `${this.webliteUrl}/databases`; + const url = `${this.webliteUrl}/databases` const response = await this.fetch(url, { method: 'GET' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`); + throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) } - return await response.json(); + return await response.json() } } -export function createClient(config: SQLiteCloudClientConfig | string) { - return new SQLiteCloudClient(config); +export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { + return new SQLiteCloudClient(config) } diff --git a/src/drivers/constants.ts b/src/drivers/constants.ts index ba82d3b..ea4403a 100644 --- a/src/drivers/constants.ts +++ b/src/drivers/constants.ts @@ -1,4 +1,3 @@ - const version = '0.0.1' let JS_ENV = '' // @ts-ignore @@ -14,8 +13,8 @@ if (typeof Deno !== 'undefined') { export const DEFAULT_HEADERS = { 'X-Client-Info': `sqlitecloud-js-${JS_ENV}/${version}` } export const DEFAULT_GLOBAL_OPTIONS = { - headers: DEFAULT_HEADERS, + headers: DEFAULT_HEADERS } export const DEFAULT_WEBLITE_VERSION = 'v2' -export const WEBLITE_PORT = 8090 \ No newline at end of file +export const WEBLITE_PORT = 8090 diff --git a/src/drivers/database.ts b/src/drivers/database.ts index 85a80de..e4033d4 100644 --- a/src/drivers/database.ts +++ b/src/drivers/database.ts @@ -14,7 +14,6 @@ import { popCallback } from './utilities' import { ErrorCallback, ResultsCallback, RowCallback, RowsCallback } from './types' import EventEmitter from 'eventemitter3' import { isBrowser } from './utilities' -import { PubSub } from './pubsub' import { Statement } from './statement' // Uses eventemitter3 instead of node events for browser compatibility @@ -479,24 +478,4 @@ export class Database extends EventEmitter { }) }) } - - /** - * PubSub class provides a Pub/Sub real-time updates and notifications system to - * allow multiple applications to communicate with each other asynchronously. - * It allows applications to subscribe to tables and receive notifications whenever - * data changes in the database table. It also enables sending messages to anyone - * subscribed to a specific channel. - * @returns {PubSub} A PubSub object - */ - public async getPubSub(): Promise { - return new Promise((resolve, reject) => { - this.getConnection((error, connection) => { - if (error || !connection) { - reject(error) - } else { - resolve(new PubSub(connection)) - } - }) - }) - } } diff --git a/src/drivers/fetch.ts b/src/drivers/fetch.ts index f168ad9..b32ae6b 100644 --- a/src/drivers/fetch.ts +++ b/src/drivers/fetch.ts @@ -1,4 +1,3 @@ - import nodeFetch, { Headers as NodeFetchHeaders } from 'node-fetch' export type Fetch = typeof fetch @@ -23,14 +22,12 @@ export const resolveHeadersConstructor = () => { return Headers } - export const fetchWithAuth = (authorization: string, customFetch?: Fetch): Fetch => { - const fetch = resolveFetch(customFetch); - const HeadersConstructor = resolveHeadersConstructor(); - + const fetch = resolveFetch(customFetch) + const HeadersConstructor = resolveHeadersConstructor() return async (input, init) => { - const headers = new HeadersConstructor(init?.headers); + const headers = new HeadersConstructor(init?.headers) if (!headers.has('Authorization')) { headers.set('Authorization', `Bearer ${authorization}`) } diff --git a/src/drivers/pubsub-refactor.ts b/src/drivers/pubsub-refactor.ts new file mode 100644 index 0000000..3a61ed8 --- /dev/null +++ b/src/drivers/pubsub-refactor.ts @@ -0,0 +1,147 @@ +import { SQLiteCloudConnection } from './connection' +import SQLiteCloudTlsConnection from './connection-tls' +import { Database } from './database' +import { PubSubRefactorCallback, SQLiteCloudConfig } from './types' + +export interface PubSubOptions { + tableName?: string + channelName?: string + dbName?: string +} + +export enum PUBSUB_ENTITY_TYPE { + TABLE = 'TABLE', + CHANNEL = 'CHANNEL' +} + +const entityTypeModifiers = { + [PUBSUB_ENTITY_TYPE.TABLE]: 'TABLE', + [PUBSUB_ENTITY_TYPE.CHANNEL]: '' +} +const getDbFromConfig = (config: SQLiteCloudConfig) => new URL(config.connectionstring ?? '')?.pathname.split('/').pop() ?? '' +const formatCommand = (arr: string[]) => arr.reduce((acc, curr) => curr.length > 0 ? (acc + ' ' + curr) : acc, '') + ';' + +/** + * Pub/Sub class to receive changes on database tables or to send messages to channels. + */ +export class PubSub { + // instantiate in createConnection? + constructor(queryConnection: Database) { + this.queryConnection = queryConnection + this._pubSubConnection = null + this.defaultDatabaseName = getDbFromConfig(queryConnection.getConfiguration()) + } + + private queryConnection: Database + private _pubSubConnection: SQLiteCloudConnection | null + private defaultDatabaseName: string + /** + * Listen to a channel and start to receive messages to the provided callback. + * @param options Options for the listen operation. If tablename and channelName are provided, channelName is used. + * If no options are provided, the default database name is used. + * @param callback Callback to be called when a message is received + */ + + private get pubSubConnection(): SQLiteCloudConnection { + if (!this._pubSubConnection) { + this._pubSubConnection = new SQLiteCloudTlsConnection(this.queryConnection.getConfiguration()) + } + return this._pubSubConnection + } + + public async listen(options: PubSubOptions, callback: PubSubRefactorCallback): Promise { + const _dbName = options.dbName ? options.dbName : this.defaultDatabaseName; + const [entityType, entityName] = options.channelName + ? [PUBSUB_ENTITY_TYPE.CHANNEL, options.channelName] : [PUBSUB_ENTITY_TYPE.TABLE, options.tableName] + if (!entityName) { + throw new Error('Must provide a channelName or tableName') + } + const pubSubEntityTypeModifier = entityTypeModifiers[entityType] + + + const authCommand: string = await this.queryConnection.sql(formatCommand(['LISTEN', pubSubEntityTypeModifier, entityName, 'DATABASE', _dbName])) + + return new Promise((resolve, reject) => { + this.pubSubConnection.sendCommands(authCommand, (error, results) => { + if (error) { + callback.call(this, error, null) + reject(error) + } else { + // skip results from pubSub auth command + if (results !== 'OK') { + callback.call(this, null, results) + } + resolve(results) + } + }) + }) + } + + /** + * Stop receive messages from a table or channel. + * @param entityType One of TABLE or CHANNEL + * @param entityName Name of the table or the channel + */ + public unlisten(options: PubSubOptions): Promise { + // type this output + const [entityType, entityName] = options.channelName ? [PUBSUB_ENTITY_TYPE.CHANNEL, options.channelName] : [PUBSUB_ENTITY_TYPE.TABLE, options.tableName] + const entityTypeModifier = entityTypeModifiers[entityType] + + return this.queryConnection.sql(`UNLISTEN ${entityTypeModifier} ?;`, entityName) + } + + /** + * Create a channel to send messages to. + * @param name Channel name + * @param failIfExists Raise an error if the channel already exists + */ + public async createChannel(name: string, failIfExists: boolean = true): Promise { + // type this output + return await this.queryConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, name) + } + + /** + * Deletes a Pub/Sub channel. + * @param name Channel name + */ + public async removeChannel(name: string): Promise { + // type this output + return await this.queryConnection.sql(`REMOVE CHANNEL ?;`, name) + } + + /** + * Send a message to the channel. + */ + public notifyChannel(channelName: string, message: string): Promise { + // type this output + return this.queryConnection.sql`NOTIFY ${channelName} ${message};` + } + + /** + * Ask the server to close the connection to the database and + * to keep only open the Pub/Sub connection. + * Only interaction with Pub/Sub commands will be allowed. + */ + public setPubSubOnly(): Promise { + return new Promise((resolve, reject) => { + this.pubSubConnection.sendCommands('PUBSUB ONLY;', (error, results) => { + if (error) { + reject(error) + } else { + this.queryConnection.close() + resolve(results) + } + }) + }) + } + + /** True if Pub/Sub connection is open. */ + public connected(): boolean { + return this._pubSubConnection?.connected ?? false + } + + /** Close Pub/Sub connection. */ + public close(): void { + this._pubSubConnection?.close() + } +} diff --git a/src/drivers/pubsub.ts b/src/drivers/pubsub.ts index bd1ae92..0877a6d 100644 --- a/src/drivers/pubsub.ts +++ b/src/drivers/pubsub.ts @@ -2,12 +2,11 @@ import { SQLiteCloudConnection } from './connection' import SQLiteCloudTlsConnection from './connection-tls' import { PubSubCallback } from './types' -interface PubSubOptions { - tableName: string - dbName?: string +export enum PUBSUB_ENTITY_TYPE { + TABLE = 'TABLE', + CHANNEL = 'CHANNEL' } - /** * Pub/Sub class to receive changes on database tables or to send messages to channels. */ @@ -21,31 +20,27 @@ export class PubSub { private connectionPubSub: SQLiteCloudConnection /** - * Listen to a channel and start to receive messages to the provided callback. - * @param options Options for the listen operation + * Listen for a table or channel and start to receive messages to the provided callback. + * @param entityType One of TABLE or CHANNEL' + * @param entityName Name of the table or the channel * @param callback Callback to be called when a message is received + * @param data Extra data to be passed to the callback */ + public async listen(entityType: PUBSUB_ENTITY_TYPE, entityName: string, callback: PubSubCallback, data?: any): Promise { + // should not force user to import and pass in entity type + const entity = entityType === 'TABLE' ? 'TABLE ' : '' // should use PUBSUB_ENTITY_TYPE for check - public async listen(options: PubSubOptions, callback: PubSubCallback): Promise { - if (options.dbName) { - try { - await this.connection.sql(`USE DATABASE ${options.dbName};`) - } catch (error) { - console.error(error) - } - } - - const authCommand: string = await this.connection.sql(`LISTEN TABLE ${options.tableName};`) + const authCommand: string = await this.connection.sql(`LISTEN ${entity}${entityName};`) return new Promise((resolve, reject) => { this.connectionPubSub.sendCommands(authCommand, (error, results) => { if (error) { - callback.call(this, error, null) + callback.call(this, error, null, data) reject(error) } else { // skip results from pubSub auth command if (results !== 'OK') { - callback.call(this, null, results) + callback.call(this, null, results, data) } resolve(results) } diff --git a/src/drivers/types.ts b/src/drivers/types.ts index c92004c..6e1fa59 100644 --- a/src/drivers/types.ts +++ b/src/drivers/types.ts @@ -135,7 +135,8 @@ export type ResultsCallback = (error: Error | null, results?: T) => voi export type RowsCallback> = (error: Error | null, rows?: T[]) => void export type RowCallback> = (error: Error | null, row?: T) => void export type RowCountCallback = (error: Error | null, rowCount?: number) => void -export type PubSubCallback = (error: Error | null, results?: T) => void +export type PubSubCallback = (error: Error | null, results?: T, data?: any) => void +export type PubSubRefactorCallback = (error: Error | null, results?: T) => void /** * Certain responses include arrays with various types of metadata. @@ -160,3 +161,7 @@ export enum SQLiteCloudArrayType { ARRAY_TYPE_SQLITE_STATUS = 50 // used in sqlite_status } + +export type UploadOptions = { + replace?: boolean +} From 23635fb73cbbf3634f59981e6a2a8da1021bd3a5 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Mon, 23 Dec 2024 09:58:01 -0800 Subject: [PATCH 04/18] Client refactor --- README_Refactor.md | 9 ++ demo.ts | 112 ++++++++++++++++++ src/drivers/constants.ts | 4 +- src/drivers/types.ts | 1 - src/index.ts | 1 - src/refactor/SQLiteCloudClient.ts | 69 +++++++++++ src/refactor/SQLiteCloudFileClient.ts | 105 ++++++++++++++++ .../SQLiteCloudPubSubClient.ts} | 88 ++++++++------ .../SQLiteCloudWebliteClient.ts} | 79 ++++-------- src/{drivers => refactor}/fetch.ts | 1 + src/refactor/utils/index.ts | 25 ++++ 11 files changed, 394 insertions(+), 100 deletions(-) create mode 100644 README_Refactor.md create mode 100644 demo.ts create mode 100644 src/refactor/SQLiteCloudClient.ts create mode 100644 src/refactor/SQLiteCloudFileClient.ts rename src/{drivers/pubsub-refactor.ts => refactor/SQLiteCloudPubSubClient.ts} (57%) rename src/{SQLiteCloudClient.ts => refactor/SQLiteCloudWebliteClient.ts} (54%) rename src/{drivers => refactor}/fetch.ts (96%) create mode 100644 src/refactor/utils/index.ts diff --git a/README_Refactor.md b/README_Refactor.md new file mode 100644 index 0000000..1dc6826 --- /dev/null +++ b/README_Refactor.md @@ -0,0 +1,9 @@ +Refactor Summary +- Added SQLiteCloudClient class and createClient function +- Extracted PubSub from Database to SQLiteCloudClient +- Added fetch and fetchWithAuth +- Added Weblite endpoint methods for upload, download, delete, and listDatabases +- Refactored PubSub to be more intuitive and easier to use +- Added FileClient class and methods for file upload and download + +TODO: Polish code, add error handling, Write tests \ No newline at end of file diff --git a/demo.ts b/demo.ts new file mode 100644 index 0000000..1b15907 --- /dev/null +++ b/demo.ts @@ -0,0 +1,112 @@ + +/** + * Developer experience - current + * + */ + +import { Database } from '@sqlitecloud/drivers' +import { PUBSUB_ENTITY_TYPE } from '@sqlitecloud/drivers/lib/drivers/pubsub' // forces user to import pubsub constants from hard to remember location + +const db = new Database('connection-string') +const pubSub = await db.getPubSub() // couples database to pubsub + +/* Database methods */ +await db.sql`SELECT * FROM users` +db.exec('command') +db.run('command') +db.all('command') +db.each('command') +db.close() + +/* PubSub usage */ +/** Listen to a table */ +pubSub.listen(PUBSUB_ENTITY_TYPE.TABLE, 'users', (error, results, data) => { // note extraneous "data" + console.log(error, results, data) +}, ['extra data']) + +/** Listen to a channel */ +pubSub.listen(PUBSUB_ENTITY_TYPE.CHANNEL, 'messages', (error, results, data) => { + console.log(error, results, data) +}, ['extra data']) + +/** Create a channel */ +pubSub.createChannel('messages') + +/** Unlisten to a table */ +pubSub.unlisten(PUBSUB_ENTITY_TYPE.TABLE, 'users') + +/** Remove a channel (not currently exposed) */ +// @ts-ignore +pubSub.removeChannel('messages') + +/** Notify a channel */ +pubSub.notifyChannel('messages', 'my message') + + +/** + * Developer experience - refactored + * In the refactor, Database still exists and works as before. + */ + +import { createClient } from './src/refactor/SQLiteCloudClient' + +const client = createClient('connection-string/chinook.db') + +// Promise sql query +const { data, error } = await client.sql`SELECT * FROM albums`; + +client.defaultDb = 'users'; // helper to set default database for SQL queries + +const { data: sessions, error: sessionsError } = await client.sql`SELECT * FROM sessions`; +// or +const result = client.db.exec('SELECT * FROM sessions') + +// Weblite +// upload database +const uploadDatabaseResponse = await client.weblite.upload('new_chinook.db', new File([''], 'new_chinook.db'), { replace: false }); + +// download database +const downloadDatabaseResponse = await client.weblite.download('new_chinook.db'); + +// delete database +const deleteDatabaseResponse = await client.weblite.delete('new_chinook.db'); + +// list databases +const listDatabasesResponse = await client.weblite.listDatabases(); + +// create database +const createDatabaseResponse = await client.weblite.create('new_chinook.db'); + +// SQLiteCloudFileClient +const createBucketResponse = await client.files.createBucket('myBucket'); +const getBucketResponse = await client.files.getBucket('myBucket'); +const deleteBucketResponse = await client.files.deleteBucket('myBucket'); +const listBucketsResponse = await client.files.listBuckets(); + +// upload file +const uploadFileResponse = await client.files.upload('myBucket', 'myPath', new File([''], 'myFile.txt'), { contentType: 'text/plain' }); + +// download file +const downloadFileResponse = await client.files.download('myBucket', 'myPath'); + +// remove file +const removeFileResponse = await client.files.remove('myBucket', 'myPath'); + + +// SQLiteCloudPubSubClient Refactor +await client.pubSub.create('messages') +await client.pubSub.notify('messages', 'my message') +await client.pubSub.subscribe('messages', (error, results) => { + console.log(error, results) +}) +client.pubSub.unsubscribe('messages') +await client.pubSub.delete('messages') + +await client.pubSub.listen({ tableName: 'users' }, (error, results) => { + console.log(error, results) +}) + +await client.pubSub.listen({ tableName: 'users', dbName: 'chinook.sqlite' }, (error, results) => { // note optional dbName + console.log(error, results) +}) + diff --git a/src/drivers/constants.ts b/src/drivers/constants.ts index ea4403a..154cd02 100644 --- a/src/drivers/constants.ts +++ b/src/drivers/constants.ts @@ -16,5 +16,5 @@ export const DEFAULT_GLOBAL_OPTIONS = { headers: DEFAULT_HEADERS } -export const DEFAULT_WEBLITE_VERSION = 'v2' -export const WEBLITE_PORT = 8090 +export const DEFAULT_API_VERSION = 'v2' +export const DEFAULT_API_PORT = 8090 diff --git a/src/drivers/types.ts b/src/drivers/types.ts index 6e1fa59..e5a12d4 100644 --- a/src/drivers/types.ts +++ b/src/drivers/types.ts @@ -136,7 +136,6 @@ export type RowsCallback> = (error: Error | null, rows?: export type RowCallback> = (error: Error | null, row?: T) => void export type RowCountCallback = (error: Error | null, rowCount?: number) => void export type PubSubCallback = (error: Error | null, results?: T, data?: any) => void -export type PubSubRefactorCallback = (error: Error | null, results?: T) => void /** * Certain responses include arrays with various types of metadata. diff --git a/src/index.ts b/src/index.ts index 386cbc3..3eb0db0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,4 +20,3 @@ export { export { SQLiteCloudRowset, SQLiteCloudRow } from './drivers/rowset' export { parseconnectionstring, validateConfiguration, getInitializationCommands, sanitizeSQLiteIdentifier } from './drivers/utilities' export * as protocol from './drivers/protocol' -export { createClient } from './SQLiteCloudClient' \ No newline at end of file diff --git a/src/refactor/SQLiteCloudClient.ts b/src/refactor/SQLiteCloudClient.ts new file mode 100644 index 0000000..41296fa --- /dev/null +++ b/src/refactor/SQLiteCloudClient.ts @@ -0,0 +1,69 @@ +import { Database } from '../drivers/database' +import { Fetch, fetchWithAuth } from './fetch' +import { SQLiteCloudPubSubClient } from './SQLiteCloudPubSubClient' +import { SQLiteCloudWebliteClient } from './SQLiteCloudWebliteClient' +import { SQLiteCloudFileClient } from './SQLiteCloudFileClient' +import { SQLiteCloudCommand } from '../drivers/types' +import { getDefaultDatabase } from './utils' + +interface SQLiteCloudClientConfig { + connectionString: string + fetch?: Fetch +} + + + +export class SQLiteCloudClient { + private connectionString: string + private fetch: Fetch + + constructor(config: SQLiteCloudClientConfig | string) { + let connectionString: string + let customFetch: Fetch | undefined + + if (typeof config === 'string') { + connectionString = config + } else { + connectionString = config.connectionString + customFetch = config.fetch + } + + this.connectionString = connectionString + this.fetch = fetchWithAuth(this.connectionString, customFetch) + this.defaultDb = getDefaultDatabase(this.connectionString) ?? '' + } + + async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { + this.db.exec(`USE DATABASE ${this.defaultDb}`) + try { + const result = await this.db.sql(sql, ...values) + return { data: result, error: null } + } catch (error) { + return { error, data: null } + } + } + + get pubSub() { + return new SQLiteCloudPubSubClient(this.db) + } + + get db() { + return new Database(this.connectionString) + } + + get weblite() { + return new SQLiteCloudWebliteClient(this.connectionString, this.fetch) + } + + get files() { + return new SQLiteCloudFileClient(this.connectionString, this.fetch) + } + + set defaultDb(dbName: string) { + this.defaultDb = dbName + } +} + +export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { + return new SQLiteCloudClient(config) +} diff --git a/src/refactor/SQLiteCloudFileClient.ts b/src/refactor/SQLiteCloudFileClient.ts new file mode 100644 index 0000000..9766f70 --- /dev/null +++ b/src/refactor/SQLiteCloudFileClient.ts @@ -0,0 +1,105 @@ +import { SQLiteCloudError } from "../drivers/types" +import { getAPIUrl } from "./utils" +import { Fetch, fetchWithAuth } from "./fetch" + +interface SQLiteCloudFile { + createBucket(bucket: string, path: string): Promise + getBucket(bucket: string): Promise + deleteBucket(bucket: string): Promise + listBuckets(): Promise + upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise + download(bucket: string, pathname: string): Promise + remove(bucket: string, pathName: string): Promise + list(bucket: string): Promise +} + +const FILES_DATABASE = 'files.sqlite' + +export class SQLiteCloudFileClient implements SQLiteCloudFile { + private filesUrl: string + private webliteSQLUrl: string + private fetch: Fetch + + constructor(connectionString: string, sql?: Fetch) { + this.filesUrl = getAPIUrl(connectionString, 'files') + this.webliteSQLUrl = getAPIUrl(connectionString, 'weblite/sql') + this.fetch = fetchWithAuth(connectionString, fetch) + } + + async createBucket(bucket: string) { + const url = `${this.webliteSQLUrl}?sql=USE DATABASE files; INSERT INTO files (Bucket) VALUES ('${bucket}');` + const response = await this.fetch(url, { method: 'POST' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to create bucket: ${response.statusText}`) + } + return response.json() + } + + async getBucket(bucket: string) { + const url = `${this.filesUrl}/${bucket}` + const response = await this.fetch(url, { method: 'GET' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to get bucket: ${response.statusText}`) + } + + return response.json() + } + + async deleteBucket(bucket: string) { + const url = `${this.filesUrl}/${bucket}` + const response = await this.fetch(url, { method: 'DELETE' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to delete bucket: ${response.statusText}`) + } + return response.json() + } + + async listBuckets() { + const encodedUrl = encodeURIComponent(`${this.webliteSQLUrl}?sql=USE DATABASE files.sqlite; SELECT * FROM files`) + const response = await this.fetch(encodedUrl) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list buckets: ${response.statusText}`) + } + return response.json() + } + + async upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }) { + const url = `${this.filesUrl}/${bucket}/${pathname}`; + const headers = { + 'Content-Type': options?.contentType || 'application/octet-stream' + } + const response = await this.fetch(url, { method: 'POST', body: file, headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to upload file: ${response.statusText}`) + } + return response.json() + } + + async download(bucket: string, pathname: string) { + const url = `${this.filesUrl}/${bucket}/${pathname}`; + const response = await this.fetch(url, { method: 'GET' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to download file: ${response.statusText}`) + } + return response.blob() + } + + async remove(bucket: string, pathName: string) { + const url = `${this.filesUrl}/${bucket}/${pathName}` + const response = await this.fetch(url, { method: 'DELETE' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to remove file: ${response.statusText}`) + } + return response.json() + } + + async list(bucket: string) { + const encodedUrl = encodeURIComponent(`${this.webliteSQLUrl}?sql=USE DATABASE files.sqlite; SELECT * FROM files WHERE bucket = '${bucket}'`) + const response = await this.fetch(encodedUrl) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list files: ${response.statusText}`) + } + return response.json() + } +} + diff --git a/src/drivers/pubsub-refactor.ts b/src/refactor/SQLiteCloudPubSubClient.ts similarity index 57% rename from src/drivers/pubsub-refactor.ts rename to src/refactor/SQLiteCloudPubSubClient.ts index 3a61ed8..808a467 100644 --- a/src/drivers/pubsub-refactor.ts +++ b/src/refactor/SQLiteCloudPubSubClient.ts @@ -1,30 +1,33 @@ -import { SQLiteCloudConnection } from './connection' -import SQLiteCloudTlsConnection from './connection-tls' -import { Database } from './database' -import { PubSubRefactorCallback, SQLiteCloudConfig } from './types' - -export interface PubSubOptions { - tableName?: string - channelName?: string - dbName?: string -} +import { SQLiteCloudConnection } from '../drivers/connection' +import SQLiteCloudTlsConnection from '../drivers/connection-tls' +import { Database } from '../drivers/database' +import { SQLiteCloudConfig } from '../drivers/types' +import { formatCommand, getDbFromConfig } from './utils' + +export type PubSubCallback = (error: Error | null, results?: T) => void -export enum PUBSUB_ENTITY_TYPE { - TABLE = 'TABLE', - CHANNEL = 'CHANNEL' +export interface ListenOptions { + tableName: string + dbName?: string } -const entityTypeModifiers = { - [PUBSUB_ENTITY_TYPE.TABLE]: 'TABLE', - [PUBSUB_ENTITY_TYPE.CHANNEL]: '' +interface SQLiteCloudPubSub { + listen(options: ListenOptions, callback: PubSubCallback): Promise + unlisten(options: ListenOptions): void + subscribe(channelName: string, callback: PubSubCallback): Promise + unsubscribe(channelName: string): void + create(channelName: string, failIfExists: boolean): Promise + delete(channelName: string): Promise + notify(channelName: string, message: string): Promise + setPubSubOnly(): Promise + connected(): boolean + close(): void } -const getDbFromConfig = (config: SQLiteCloudConfig) => new URL(config.connectionstring ?? '')?.pathname.split('/').pop() ?? '' -const formatCommand = (arr: string[]) => arr.reduce((acc, curr) => curr.length > 0 ? (acc + ' ' + curr) : acc, '') + ';' /** * Pub/Sub class to receive changes on database tables or to send messages to channels. */ -export class PubSub { +export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { // instantiate in createConnection? constructor(queryConnection: Database) { this.queryConnection = queryConnection @@ -49,17 +52,9 @@ export class PubSub { return this._pubSubConnection } - public async listen(options: PubSubOptions, callback: PubSubRefactorCallback): Promise { + public async listen(options: ListenOptions, callback: PubSubCallback): Promise { const _dbName = options.dbName ? options.dbName : this.defaultDatabaseName; - const [entityType, entityName] = options.channelName - ? [PUBSUB_ENTITY_TYPE.CHANNEL, options.channelName] : [PUBSUB_ENTITY_TYPE.TABLE, options.tableName] - if (!entityName) { - throw new Error('Must provide a channelName or tableName') - } - const pubSubEntityTypeModifier = entityTypeModifiers[entityType] - - - const authCommand: string = await this.queryConnection.sql(formatCommand(['LISTEN', pubSubEntityTypeModifier, entityName, 'DATABASE', _dbName])) + const authCommand: string = await this.queryConnection.sql`LISTEN ${options.tableName} DATABASE ${_dbName};` return new Promise((resolve, reject) => { this.pubSubConnection.sendCommands(authCommand, (error, results) => { @@ -82,12 +77,27 @@ export class PubSub { * @param entityType One of TABLE or CHANNEL * @param entityName Name of the table or the channel */ - public unlisten(options: PubSubOptions): Promise { - // type this output - const [entityType, entityName] = options.channelName ? [PUBSUB_ENTITY_TYPE.CHANNEL, options.channelName] : [PUBSUB_ENTITY_TYPE.TABLE, options.tableName] - const entityTypeModifier = entityTypeModifiers[entityType] + public unlisten(options: ListenOptions): void { + this.queryConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` + } + + public async subscribe(channelName: string, callback: PubSubCallback): Promise { + const authCommand: string = await this.queryConnection.sql`LISTEN ${channelName};` + + return new Promise((resolve, reject) => { + this.pubSubConnection.sendCommands(authCommand, (error, results) => { + if (error) { + callback.call(this, error, null) + reject(error) + } else { + resolve(results) + } + }) + }) + } - return this.queryConnection.sql(`UNLISTEN ${entityTypeModifier} ?;`, entityName) + public unsubscribe(channelName: string): void { + this.queryConnection.sql`UNLISTEN ${channelName};` } /** @@ -95,24 +105,24 @@ export class PubSub { * @param name Channel name * @param failIfExists Raise an error if the channel already exists */ - public async createChannel(name: string, failIfExists: boolean = true): Promise { + public async create(channelName: string, failIfExists: boolean = true): Promise { // type this output - return await this.queryConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, name) + return await this.queryConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName) } /** * Deletes a Pub/Sub channel. * @param name Channel name */ - public async removeChannel(name: string): Promise { + public async delete(channelName: string): Promise { // type this output - return await this.queryConnection.sql(`REMOVE CHANNEL ?;`, name) + return await this.queryConnection.sql(`REMOVE CHANNEL ?;`, channelName) } /** * Send a message to the channel. */ - public notifyChannel(channelName: string, message: string): Promise { + public notify(channelName: string, message: string): Promise { // type this output return this.queryConnection.sql`NOTIFY ${channelName} ${message};` } diff --git a/src/SQLiteCloudClient.ts b/src/refactor/SQLiteCloudWebliteClient.ts similarity index 54% rename from src/SQLiteCloudClient.ts rename to src/refactor/SQLiteCloudWebliteClient.ts index aa0a067..63fa876 100644 --- a/src/SQLiteCloudClient.ts +++ b/src/refactor/SQLiteCloudWebliteClient.ts @@ -1,65 +1,22 @@ -import { Database } from './drivers/database' -import { SQLiteCloudError, UploadOptions } from './drivers/types' -import { Fetch, fetchWithAuth } from './drivers/fetch' -import { DEFAULT_HEADERS, DEFAULT_WEBLITE_VERSION, WEBLITE_PORT } from './drivers/constants' -import { PubSub } from './drivers/pubsub-refactor' +import { SQLiteCloudError, UploadOptions } from '../drivers/types' +import { Fetch, fetchWithAuth } from './fetch' +import { DEFAULT_HEADERS } from '../drivers/constants' -interface SQLiteCloudClientConfig { - connectionString: string - fetch?: Fetch -} - -interface ISQLiteCloudClient { - pubSub: PubSub - db: Database - upload(databaseName: string, file: File | Buffer | Blob | string, opts?: UploadOptions): Promise +interface SQLiteCloudWeblite { + upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise download(databaseName: string): Promise delete(databaseName: string): Promise listDatabases(): Promise + create(databaseName: string): Promise } -const parseConnectionString = (connectionString: string) => { - const url = new URL(connectionString) - return { - host: url.hostname, - port: url.port, - database: url.pathname.slice(1), - apiKey: url.searchParams.get('apikey') - } -} - -export class SQLiteCloudClient implements ISQLiteCloudClient { - // TODO: Add support for custom fetch - private fetch: Fetch - private connectionString: string +export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { private webliteUrl: string - private _db: Database - private _pubSub: PubSub - - constructor(config: SQLiteCloudClientConfig | string) { - let connectionString: string - if (typeof config === 'string') { - connectionString = config - } else { - connectionString = config.connectionString - } - - this.connectionString = connectionString - this._db = new Database(this.connectionString) - this._pubSub = new PubSub(this.db) - this.fetch = fetchWithAuth(this.connectionString) - - const { host } = parseConnectionString(this.connectionString) - - this.webliteUrl = `https://${host}:${WEBLITE_PORT}/${DEFAULT_WEBLITE_VERSION}/weblite` - } - - get pubSub() { - return this._pubSub - } + private fetch: Fetch - get db() { - return this._db + constructor(connectionString: string, fetch?: Fetch) { + this.webliteUrl = getAPIUrl(connectionString, 'weblite') + this.fetch = fetch || fetchWithAuth(connectionString) } async upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions = {}) { @@ -119,8 +76,16 @@ export class SQLiteCloudClient implements ISQLiteCloudClient { } return await response.json() } -} -export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { - return new SQLiteCloudClient(config) + async create(databaseName: string) { + const response = await fetch(`${this.webliteUrl}/sql?sql=CREATE DATABASE ${databaseName}`, { method: 'POST' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to create database: ${response.statusText}`) + } + return response + } } +function getAPIUrl(connectionString: string, arg1: string): string { + throw new Error('Function not implemented.') +} + diff --git a/src/drivers/fetch.ts b/src/refactor/fetch.ts similarity index 96% rename from src/drivers/fetch.ts rename to src/refactor/fetch.ts index b32ae6b..d51dc7e 100644 --- a/src/drivers/fetch.ts +++ b/src/refactor/fetch.ts @@ -22,6 +22,7 @@ export const resolveHeadersConstructor = () => { return Headers } +// authorization is the connection string export const fetchWithAuth = (authorization: string, customFetch?: Fetch): Fetch => { const fetch = resolveFetch(customFetch) const HeadersConstructor = resolveHeadersConstructor() diff --git a/src/refactor/utils/index.ts b/src/refactor/utils/index.ts new file mode 100644 index 0000000..3b976ab --- /dev/null +++ b/src/refactor/utils/index.ts @@ -0,0 +1,25 @@ +import { DEFAULT_API_PORT, DEFAULT_API_VERSION } from "../../drivers/constants" +import { SQLiteCloudConfig } from "../../drivers/types" + +export const parseConnectionString = (connectionString: string) => { + const url = new URL(connectionString) + return { + host: url.hostname, + port: url.port, + database: url.pathname.slice(1), + apiKey: url.searchParams.get('apikey') + } +} + +export const getAPIUrl = (connectionString: string, path: string) => { + const { host } = parseConnectionString(connectionString) + return `https://${host}:${DEFAULT_API_PORT}/${DEFAULT_API_VERSION}/${path}` +} + +export const getDefaultDatabase = (connectionString: string) => { + const { database } = parseConnectionString(connectionString) + return database +} + +export const getDbFromConfig = (config: SQLiteCloudConfig) => new URL(config.connectionstring ?? '')?.pathname.split('/').pop() ?? '' +export const formatCommand = (arr: string[]) => arr.reduce((acc, curr) => curr.length > 0 ? (acc + ' ' + curr) : acc, '') + ';' From 988790f4578e0fd74ec997ae5c63775b27bc311e Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Mon, 23 Dec 2024 10:15:58 -0800 Subject: [PATCH 05/18] fix: update SQLiteCloudClient to use _db --- src/refactor/SQLiteCloudClient.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/refactor/SQLiteCloudClient.ts b/src/refactor/SQLiteCloudClient.ts index 41296fa..bf793f3 100644 --- a/src/refactor/SQLiteCloudClient.ts +++ b/src/refactor/SQLiteCloudClient.ts @@ -16,6 +16,7 @@ interface SQLiteCloudClientConfig { export class SQLiteCloudClient { private connectionString: string private fetch: Fetch + private _db: Database constructor(config: SQLiteCloudClientConfig | string) { let connectionString: string @@ -31,6 +32,7 @@ export class SQLiteCloudClient { this.connectionString = connectionString this.fetch = fetchWithAuth(this.connectionString, customFetch) this.defaultDb = getDefaultDatabase(this.connectionString) ?? '' + this._db = new Database(this.connectionString) } async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { @@ -48,7 +50,7 @@ export class SQLiteCloudClient { } get db() { - return new Database(this.connectionString) + return this._db } get weblite() { From d6c109b0ac80aa73557248c306dca8eea28b9779 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Mon, 23 Dec 2024 12:45:36 -0800 Subject: [PATCH 06/18] add vector client scaffold --- src/refactor/SQLiteCloudVectorClient.ts | 96 +++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/refactor/SQLiteCloudVectorClient.ts diff --git a/src/refactor/SQLiteCloudVectorClient.ts b/src/refactor/SQLiteCloudVectorClient.ts new file mode 100644 index 0000000..734373b --- /dev/null +++ b/src/refactor/SQLiteCloudVectorClient.ts @@ -0,0 +1,96 @@ +import { Database } from "../drivers/database"; + +interface Column { + name: string; + type: string; + partitionKey?: boolean; + primaryKey?: boolean; +} + +interface IndexOptions { + tableName: string; + dimensions: number; + columns: Column[]; + binaryQuantization?: boolean; + dbName?: string; +} + +type UpsertData = [Record & { id: string | number }][] + +interface QueryOptions { + topK: number, + where?: string[] +} + +interface SQLiteCloudVector { + init(options: IndexOptions): Promise + upsert(data: UpsertData): Promise + query(queryEmbedding: number[], options: QueryOptions): Promise +} + +const DEFAULT_EMBEDDING_COLUMN_NAME = 'embedding' + +const buildEmbeddingType = (dimensions: number, binaryQuantization: boolean) => { + return `${binaryQuantization ? 'BIT' : 'FLOAT'}[${dimensions}]` +} + +const formatInitColumns = (opts: IndexOptions) => { + const { columns, dimensions, binaryQuantization } = opts + return columns.reduce((acc, column) => { + let _type = column.type.toLowerCase(); + const { name, primaryKey, partitionKey } = column + if (_type === 'embedding') { + _type = buildEmbeddingType(dimensions, !!binaryQuantization) + } + const formattedColumn = `${name} ${_type} ${primaryKey ? 'PRIMARY KEY' : ''}${partitionKey ? 'PARTITION KEY' : ''}` + return `${acc}, ${formattedColumn}` + }, '') +} + +function formatUpsertCommand(data: UpsertData): [any, any] { + throw new Error("Function not implemented."); +} + + +export class SQLiteCloudVectorClient implements SQLiteCloudVector { + + private _db: Database + private _tableName: string + private _columns: Column[] + private _formattedColumns: string + + constructor(_db: Database) { + this._db = _db + this._tableName = '' + this._columns = [] + this._formattedColumns = '' + } + + async init(options: IndexOptions) { + const formattedColumns = formatInitColumns(options) + this._tableName = options.tableName + this._columns = options?.columns || [] + this._formattedColumns = formattedColumns + const useDbCommand = options?.dbName ? `USE DATABASE ${options.dbName}; ` : '' + const hasTable = await this._db.sql`${useDbCommand}SELECT 1 FROM ${options.tableName} LIMIT 1;` + + if (hasTable.length === 0) { // TODO - VERIFY CHECK HAS TABLE + const query = `CREATE VIRTUAL TABLE ${options.tableName} USING vec0(${formattedColumns})` + await this._db.sql(query) + } + return this + } + + async upsert(data: UpsertData) { + const [formattedColumns, formattedValues] = formatUpsertCommand(data) + const query = `INSERT INTO ${this._tableName}(${formattedColumns}) VALUES (${formattedValues})` + return await this._db.sql(query) + } + + async query(queryEmbedding: number[], options: QueryOptions) { + const query = `SELECT * FROM ${this._tableName} WHERE ${DEFAULT_EMBEDDING_COLUMN_NAME} match ${JSON.stringify(queryEmbedding)} and k = ${options.topK} and ${(options?.where?.join(' and ') || '')}` + const result = await this._db.sql(query) + return { data: result, error: null } + } + +} From f28bab695055d65397002e817f7df92b34666398 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Tue, 24 Dec 2024 13:53:47 -0800 Subject: [PATCH 07/18] reorg into packages dir --- README_Refactor.md | 23 +++++++++++++++---- demo.ts | 2 +- package.json | 2 +- src/{refactor => }/SQLiteCloudClient.ts | 16 ++++++------- .../pubsub}/SQLiteCloudPubSubClient.ts | 9 ++++---- .../storage}/SQLiteCloudFileClient.ts | 6 ++--- src/packages/test/SQLiteCloudClient.test.ts | 0 src/{refactor => packages/utils}/fetch.ts | 0 src/{refactor => packages}/utils/index.ts | 0 .../vector}/SQLiteCloudVectorClient.ts | 2 +- .../weblite}/SQLiteCloudWebliteClient.ts | 10 ++++---- 11 files changed, 40 insertions(+), 30 deletions(-) rename src/{refactor => }/SQLiteCloudClient.ts (76%) rename src/{refactor => packages/pubsub}/SQLiteCloudPubSubClient.ts (94%) rename src/{refactor => packages/storage}/SQLiteCloudFileClient.ts (96%) create mode 100644 src/packages/test/SQLiteCloudClient.test.ts rename src/{refactor => packages/utils}/fetch.ts (100%) rename src/{refactor => packages}/utils/index.ts (100%) rename src/{refactor => packages/vector}/SQLiteCloudVectorClient.ts (98%) rename src/{refactor => packages/weblite}/SQLiteCloudWebliteClient.ts (90%) diff --git a/README_Refactor.md b/README_Refactor.md index 1dc6826..1baee58 100644 --- a/README_Refactor.md +++ b/README_Refactor.md @@ -1,9 +1,24 @@ -Refactor Summary +Refactor 12.23.24 - Added SQLiteCloudClient class and createClient function - Extracted PubSub from Database to SQLiteCloudClient -- Added fetch and fetchWithAuth -- Added Weblite endpoint methods for upload, download, delete, and listDatabases +- Added fetch, customFetch support and fetchWithAuth +- Added Weblite methods for upload, download, delete, and listDatabases - Refactored PubSub to be more intuitive and easier to use - Added FileClient class and methods for file upload and download +- Added SQLiteCloudVectorClient class and methods for upsert and query -TODO: Polish code, add error handling, Write tests \ No newline at end of file +Refactor 12.24.14 +Scope of work: +- refactor new code to improve code smells and readability + - Recap progress. + - write tests for each new class + - Idenitfy protential issues + - Plan refactor with psuedo code + - Implement refactor + - Test refactor + +TODO: +- add error handling and logging +- add tests +- add comments +- add documentation \ No newline at end of file diff --git a/demo.ts b/demo.ts index 1b15907..ca9621a 100644 --- a/demo.ts +++ b/demo.ts @@ -48,7 +48,7 @@ pubSub.notifyChannel('messages', 'my message') * In the refactor, Database still exists and works as before. */ -import { createClient } from './src/refactor/SQLiteCloudClient' +import { createClient } from './src/SQLiteCloudClient' const client = createClient('connection-string/chinook.db') diff --git a/package.json b/package.json index ffea1e1..ae196c8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.357", + "version": "1.0.358", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/refactor/SQLiteCloudClient.ts b/src/SQLiteCloudClient.ts similarity index 76% rename from src/refactor/SQLiteCloudClient.ts rename to src/SQLiteCloudClient.ts index bf793f3..ac47c4b 100644 --- a/src/refactor/SQLiteCloudClient.ts +++ b/src/SQLiteCloudClient.ts @@ -1,18 +1,16 @@ -import { Database } from '../drivers/database' -import { Fetch, fetchWithAuth } from './fetch' -import { SQLiteCloudPubSubClient } from './SQLiteCloudPubSubClient' -import { SQLiteCloudWebliteClient } from './SQLiteCloudWebliteClient' -import { SQLiteCloudFileClient } from './SQLiteCloudFileClient' -import { SQLiteCloudCommand } from '../drivers/types' -import { getDefaultDatabase } from './utils' +import { Database } from './drivers/database' +import { Fetch, fetchWithAuth } from './packages/utils/fetch' +import { SQLiteCloudPubSubClient } from './packages/pubsub/SQLiteCloudPubSubClient' +import { SQLiteCloudWebliteClient } from './packages/weblite/SQLiteCloudWebliteClient' +import { SQLiteCloudFileClient } from './packages/storage/SQLiteCloudFileClient' +import { SQLiteCloudCommand } from './drivers/types' +import { getDefaultDatabase } from './packages/utils' interface SQLiteCloudClientConfig { connectionString: string fetch?: Fetch } - - export class SQLiteCloudClient { private connectionString: string private fetch: Fetch diff --git a/src/refactor/SQLiteCloudPubSubClient.ts b/src/packages/pubsub/SQLiteCloudPubSubClient.ts similarity index 94% rename from src/refactor/SQLiteCloudPubSubClient.ts rename to src/packages/pubsub/SQLiteCloudPubSubClient.ts index 808a467..6ecb180 100644 --- a/src/refactor/SQLiteCloudPubSubClient.ts +++ b/src/packages/pubsub/SQLiteCloudPubSubClient.ts @@ -1,8 +1,7 @@ -import { SQLiteCloudConnection } from '../drivers/connection' -import SQLiteCloudTlsConnection from '../drivers/connection-tls' -import { Database } from '../drivers/database' -import { SQLiteCloudConfig } from '../drivers/types' -import { formatCommand, getDbFromConfig } from './utils' +import { SQLiteCloudConnection } from '../../drivers/connection' +import SQLiteCloudTlsConnection from '../../drivers/connection-tls' +import { Database } from '../../drivers/database' +import { getDbFromConfig } from '../utils' export type PubSubCallback = (error: Error | null, results?: T) => void diff --git a/src/refactor/SQLiteCloudFileClient.ts b/src/packages/storage/SQLiteCloudFileClient.ts similarity index 96% rename from src/refactor/SQLiteCloudFileClient.ts rename to src/packages/storage/SQLiteCloudFileClient.ts index 9766f70..fd63505 100644 --- a/src/refactor/SQLiteCloudFileClient.ts +++ b/src/packages/storage/SQLiteCloudFileClient.ts @@ -1,6 +1,6 @@ -import { SQLiteCloudError } from "../drivers/types" -import { getAPIUrl } from "./utils" -import { Fetch, fetchWithAuth } from "./fetch" +import { SQLiteCloudError } from "../../drivers/types" +import { getAPIUrl } from "../utils" +import { Fetch, fetchWithAuth } from "../utils/fetch" interface SQLiteCloudFile { createBucket(bucket: string, path: string): Promise diff --git a/src/packages/test/SQLiteCloudClient.test.ts b/src/packages/test/SQLiteCloudClient.test.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/refactor/fetch.ts b/src/packages/utils/fetch.ts similarity index 100% rename from src/refactor/fetch.ts rename to src/packages/utils/fetch.ts diff --git a/src/refactor/utils/index.ts b/src/packages/utils/index.ts similarity index 100% rename from src/refactor/utils/index.ts rename to src/packages/utils/index.ts diff --git a/src/refactor/SQLiteCloudVectorClient.ts b/src/packages/vector/SQLiteCloudVectorClient.ts similarity index 98% rename from src/refactor/SQLiteCloudVectorClient.ts rename to src/packages/vector/SQLiteCloudVectorClient.ts index 734373b..d56f80e 100644 --- a/src/refactor/SQLiteCloudVectorClient.ts +++ b/src/packages/vector/SQLiteCloudVectorClient.ts @@ -1,4 +1,4 @@ -import { Database } from "../drivers/database"; +import { Database } from "../../drivers/database"; interface Column { name: string; diff --git a/src/refactor/SQLiteCloudWebliteClient.ts b/src/packages/weblite/SQLiteCloudWebliteClient.ts similarity index 90% rename from src/refactor/SQLiteCloudWebliteClient.ts rename to src/packages/weblite/SQLiteCloudWebliteClient.ts index 63fa876..4a163d0 100644 --- a/src/refactor/SQLiteCloudWebliteClient.ts +++ b/src/packages/weblite/SQLiteCloudWebliteClient.ts @@ -1,6 +1,7 @@ -import { SQLiteCloudError, UploadOptions } from '../drivers/types' -import { Fetch, fetchWithAuth } from './fetch' -import { DEFAULT_HEADERS } from '../drivers/constants' +import { SQLiteCloudError, UploadOptions } from '../../drivers/types' +import { Fetch, fetchWithAuth } from '../utils/fetch' +import { DEFAULT_HEADERS } from '../../drivers/constants' +import { getAPIUrl } from '../utils' interface SQLiteCloudWeblite { upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise @@ -85,7 +86,4 @@ export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { return response } } -function getAPIUrl(connectionString: string, arg1: string): string { - throw new Error('Function not implemented.') -} From 27105e16e0f1c3ccbb53232fc1afe9dd598608e8 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Tue, 24 Dec 2024 17:35:51 -0800 Subject: [PATCH 08/18] refactor to clean up packages and start on functions client --- README_Refactor.md | 42 +++++++- demo.ts | 2 +- package.json | 2 +- src/SQLiteCloudClient.ts | 69 ------------- src/drivers/constants.ts | 5 +- src/drivers/types.ts | 1 + src/packages/SQLiteCloudClient.ts | 81 ++++++++++++++++ src/packages/constants/index.ts | 2 + src/packages/functions/FunctionsClient.ts | 36 +++++++ ...teCloudPubSubClient.ts => PubSubClient.ts} | 41 ++++---- ...eClient.ts => SQLiteCloudStorageClient.ts} | 15 ++- src/packages/utils/index.ts | 29 ++++++ .../weblite/SQLiteCloudWebliteClient.ts | 97 +++++++++++-------- .../test => test}/SQLiteCloudClient.test.ts | 0 14 files changed, 276 insertions(+), 146 deletions(-) delete mode 100644 src/SQLiteCloudClient.ts create mode 100644 src/packages/SQLiteCloudClient.ts create mode 100644 src/packages/constants/index.ts create mode 100644 src/packages/functions/FunctionsClient.ts rename src/packages/pubsub/{SQLiteCloudPubSubClient.ts => PubSubClient.ts} (75%) rename src/packages/storage/{SQLiteCloudFileClient.ts => SQLiteCloudStorageClient.ts} (90%) rename {src/packages/test => test}/SQLiteCloudClient.test.ts (100%) diff --git a/README_Refactor.md b/README_Refactor.md index 1baee58..dd37810 100644 --- a/README_Refactor.md +++ b/README_Refactor.md @@ -7,10 +7,43 @@ Refactor 12.23.24 - Added FileClient class and methods for file upload and download - Added SQLiteCloudVectorClient class and methods for upsert and query -Refactor 12.24.14 +Refactor Summary +- The Plan: + - Improve the usability of the SQLite Cloud platform by consolidating various features + under a single client with one consistent design and interface +The Objective: + - Provide a streamlined and consistent api for discovering, learning and using features on SQLite Cloud + - Improve the visibility of various features on the SQLite Cloud platform by providing explicit namespaces and methods for: + - functions + - file storage + - Pub/Sub + - Vector search + - Weblite (platform-level database management) + - database (core database connection) + - Increase adoption of SQLite Cloud's JS SDK by expanding our documentation. + - Provide a solid architecture for future SDK development. + - Goals: + - Streamline the onboarding and implementation process for building JS apps on SQLite Cloud + +Guidelines: + - Use consistent and scalable designs to improve readability, usability and maintainability. + Scope of work: - refactor new code to improve code smells and readability - Recap progress. + - packages + - functions: + - Purpose: used to interact with edge functions deployed on the SQLite Cloud platform + - Value: removes need for custom client + - Objective: simplify the onboarding and use of edge functions to increase adoption + - storage: + - Purpose: used to store files, with an emphasis on images and photos + - Value: unifies development experience of handling transactional and non-transactional data + - Objective: simplify the development process + - pubsub: + - Purpose: used to interact with the SQLite Cloud pubsub platform + - Value: removes need for custom client + - Objective: simplify the onboarding and use of pubsub to increase adoption - write tests for each new class - Idenitfy protential issues - Plan refactor with psuedo code @@ -21,4 +54,9 @@ TODO: - add error handling and logging - add tests - add comments -- add documentation \ No newline at end of file +- add documentation + + +Out of scope: +- Auth module (awaiting auth.js merge) +- Vector search module \ No newline at end of file diff --git a/demo.ts b/demo.ts index ca9621a..8c27f1d 100644 --- a/demo.ts +++ b/demo.ts @@ -48,7 +48,7 @@ pubSub.notifyChannel('messages', 'my message') * In the refactor, Database still exists and works as before. */ -import { createClient } from './src/SQLiteCloudClient' +import { createClient } from './src/packages/SQLiteCloudClient' const client = createClient('connection-string/chinook.db') diff --git a/package.json b/package.json index ae196c8..f63491f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.358", + "version": "1.0.359", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/SQLiteCloudClient.ts b/src/SQLiteCloudClient.ts deleted file mode 100644 index ac47c4b..0000000 --- a/src/SQLiteCloudClient.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Database } from './drivers/database' -import { Fetch, fetchWithAuth } from './packages/utils/fetch' -import { SQLiteCloudPubSubClient } from './packages/pubsub/SQLiteCloudPubSubClient' -import { SQLiteCloudWebliteClient } from './packages/weblite/SQLiteCloudWebliteClient' -import { SQLiteCloudFileClient } from './packages/storage/SQLiteCloudFileClient' -import { SQLiteCloudCommand } from './drivers/types' -import { getDefaultDatabase } from './packages/utils' - -interface SQLiteCloudClientConfig { - connectionString: string - fetch?: Fetch -} - -export class SQLiteCloudClient { - private connectionString: string - private fetch: Fetch - private _db: Database - - constructor(config: SQLiteCloudClientConfig | string) { - let connectionString: string - let customFetch: Fetch | undefined - - if (typeof config === 'string') { - connectionString = config - } else { - connectionString = config.connectionString - customFetch = config.fetch - } - - this.connectionString = connectionString - this.fetch = fetchWithAuth(this.connectionString, customFetch) - this.defaultDb = getDefaultDatabase(this.connectionString) ?? '' - this._db = new Database(this.connectionString) - } - - async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { - this.db.exec(`USE DATABASE ${this.defaultDb}`) - try { - const result = await this.db.sql(sql, ...values) - return { data: result, error: null } - } catch (error) { - return { error, data: null } - } - } - - get pubSub() { - return new SQLiteCloudPubSubClient(this.db) - } - - get db() { - return this._db - } - - get weblite() { - return new SQLiteCloudWebliteClient(this.connectionString, this.fetch) - } - - get files() { - return new SQLiteCloudFileClient(this.connectionString, this.fetch) - } - - set defaultDb(dbName: string) { - this.defaultDb = dbName - } -} - -export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { - return new SQLiteCloudClient(config) -} diff --git a/src/drivers/constants.ts b/src/drivers/constants.ts index 154cd02..fa6bea9 100644 --- a/src/drivers/constants.ts +++ b/src/drivers/constants.ts @@ -11,7 +11,10 @@ if (typeof Deno !== 'undefined') { JS_ENV = 'node' } -export const DEFAULT_HEADERS = { 'X-Client-Info': `sqlitecloud-js-${JS_ENV}/${version}` } +export const DEFAULT_HEADERS = { + 'X-Client-Info': `sqlitecloud-js-${JS_ENV}/${version}`, + 'Content-Type': 'application/octet-stream' +} export const DEFAULT_GLOBAL_OPTIONS = { headers: DEFAULT_HEADERS } diff --git a/src/drivers/types.ts b/src/drivers/types.ts index e5a12d4..7d8cdb4 100644 --- a/src/drivers/types.ts +++ b/src/drivers/types.ts @@ -163,4 +163,5 @@ export enum SQLiteCloudArrayType { export type UploadOptions = { replace?: boolean + headers?: Record } diff --git a/src/packages/SQLiteCloudClient.ts b/src/packages/SQLiteCloudClient.ts new file mode 100644 index 0000000..1c8c91d --- /dev/null +++ b/src/packages/SQLiteCloudClient.ts @@ -0,0 +1,81 @@ +import { Database } from '../drivers/database' +import { Fetch, fetchWithAuth } from './utils/fetch' +import { PubSubClient } from './pubsub/PubSubClient' +import { SQLiteCloudWebliteClient } from './weblite/SQLiteCloudWebliteClient' +import { StorageClient } from './storage/SQLiteCloudStorageClient' +import { SQLiteCloudCommand, SQLiteCloudError } from '../drivers/types' +import { cleanConnectionString, getDefaultDatabase } from './utils' + +interface SQLiteCloudClientConfig { + connectionString: string + fetch?: Fetch +} + +export class SQLiteCloudClient { + protected connectionString: string + protected fetch: Fetch + protected _db: Database + + constructor(config: SQLiteCloudClientConfig | string) { + try { + if (!config) { + throw new SQLiteCloudError('Invalid connection string or config') + } + let connectionString: string + let customFetch: Fetch | undefined + + if (typeof config === 'string') { + connectionString = cleanConnectionString(config) + } else { + connectionString = config.connectionString + customFetch = config.fetch + } + + this.connectionString = connectionString + this.fetch = fetchWithAuth(this.connectionString, customFetch) + this.defaultDb = getDefaultDatabase(this.connectionString) ?? '' + this._db = new Database(this.connectionString) + } catch (error) { + throw new SQLiteCloudError('failed to initialize SQLiteCloudClient') + } + } + + async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { + this.db.exec(`USE DATABASE ${this.defaultDb}`) + try { + const result = await this.db.sql(sql, ...values) + return { data: result, error: null } + } catch (error) { + return { error, data: null } + } + } + + get pubSub() { + return new PubSubClient(this.db.getConfiguration()) + } + + get db() { + return this._db + } + + get weblite() { + return new SQLiteCloudWebliteClient(this.connectionString, this.fetch) + } + + get files() { + return new StorageClient(this.connectionString, this.fetch) + } + + get functions() { + // return new SQLiteCloudFunctionsClient(this.connectionString, this.fetch) + return null + } + + set defaultDb(dbName: string) { + this.defaultDb = dbName + } +} + +export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { + return new SQLiteCloudClient(config) +} diff --git a/src/packages/constants/index.ts b/src/packages/constants/index.ts new file mode 100644 index 0000000..402e127 --- /dev/null +++ b/src/packages/constants/index.ts @@ -0,0 +1,2 @@ +export const FILES_DATABASE = 'files.sqlite' +export const FUNCTIONS_ROOT_PATH = 'functions' \ No newline at end of file diff --git a/src/packages/functions/FunctionsClient.ts b/src/packages/functions/FunctionsClient.ts new file mode 100644 index 0000000..973e2b3 --- /dev/null +++ b/src/packages/functions/FunctionsClient.ts @@ -0,0 +1,36 @@ +import { SQLiteCloudError } from '../../drivers/types' +import { FUNCTIONS_ROOT_PATH } from '../constants' +import { getAPIUrl } from '../utils' +import { Fetch, resolveFetch, resolveHeadersConstructor } from '../utils/fetch' + +export class FunctionsClient { + protected url: string + protected fetch: Fetch + protected headers: Record + + constructor( + connectionString: string, + options: { + customFetch?: Fetch, + headers?: Record + } = {} + ) { + this.url = getAPIUrl(connectionString, FUNCTIONS_ROOT_PATH) + this.fetch = resolveFetch(options.customFetch) + this.headers = options.headers ?? {} + } + // auth token is the full connection string with apikey + setAuth(token: string) { + this.headers.Authorization = `Bearer ${token}` + } + + async invoke(functionName: string, args: any[]) { + try { + // TODO IMPLEMENT + } catch (error) { + throw new SQLiteCloudError(`Failed to invoke function: ${error}`) + } + } + + +} diff --git a/src/packages/pubsub/SQLiteCloudPubSubClient.ts b/src/packages/pubsub/PubSubClient.ts similarity index 75% rename from src/packages/pubsub/SQLiteCloudPubSubClient.ts rename to src/packages/pubsub/PubSubClient.ts index 6ecb180..e8cdda7 100644 --- a/src/packages/pubsub/SQLiteCloudPubSubClient.ts +++ b/src/packages/pubsub/PubSubClient.ts @@ -1,7 +1,6 @@ import { SQLiteCloudConnection } from '../../drivers/connection' import SQLiteCloudTlsConnection from '../../drivers/connection-tls' -import { Database } from '../../drivers/database' -import { getDbFromConfig } from '../utils' +import { SQLiteCloudConfig } from '../../drivers/types' export type PubSubCallback = (error: Error | null, results?: T) => void @@ -10,7 +9,7 @@ export interface ListenOptions { dbName?: string } -interface SQLiteCloudPubSub { +export interface PubSub { listen(options: ListenOptions, callback: PubSubCallback): Promise unlisten(options: ListenOptions): void subscribe(channelName: string, callback: PubSubCallback): Promise @@ -26,17 +25,17 @@ interface SQLiteCloudPubSub { /** * Pub/Sub class to receive changes on database tables or to send messages to channels. */ -export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { +export class PubSubClient implements PubSub { + protected _pubSubConnection: SQLiteCloudConnection | null + protected defaultDatabaseName: string + protected config: SQLiteCloudConfig // instantiate in createConnection? - constructor(queryConnection: Database) { - this.queryConnection = queryConnection + constructor(config: SQLiteCloudConfig) { + this.config = config this._pubSubConnection = null - this.defaultDatabaseName = getDbFromConfig(queryConnection.getConfiguration()) + this.defaultDatabaseName = config?.database ?? '' } - private queryConnection: Database - private _pubSubConnection: SQLiteCloudConnection | null - private defaultDatabaseName: string /** * Listen to a channel and start to receive messages to the provided callback. * @param options Options for the listen operation. If tablename and channelName are provided, channelName is used. @@ -44,16 +43,16 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { * @param callback Callback to be called when a message is received */ - private get pubSubConnection(): SQLiteCloudConnection { + get pubSubConnection(): SQLiteCloudConnection { if (!this._pubSubConnection) { - this._pubSubConnection = new SQLiteCloudTlsConnection(this.queryConnection.getConfiguration()) + this._pubSubConnection = new SQLiteCloudTlsConnection(this.config) } return this._pubSubConnection } - public async listen(options: ListenOptions, callback: PubSubCallback): Promise { + async listen(options: ListenOptions, callback: PubSubCallback): Promise { const _dbName = options.dbName ? options.dbName : this.defaultDatabaseName; - const authCommand: string = await this.queryConnection.sql`LISTEN ${options.tableName} DATABASE ${_dbName};` + const authCommand: string = await this.pubSubConnection.sql`LISTEN ${options.tableName} DATABASE ${_dbName};` return new Promise((resolve, reject) => { this.pubSubConnection.sendCommands(authCommand, (error, results) => { @@ -77,11 +76,11 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { * @param entityName Name of the table or the channel */ public unlisten(options: ListenOptions): void { - this.queryConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` + this.pubSubConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` } public async subscribe(channelName: string, callback: PubSubCallback): Promise { - const authCommand: string = await this.queryConnection.sql`LISTEN ${channelName};` + const authCommand: string = await this.pubSubConnection.sql`LISTEN ${channelName};` return new Promise((resolve, reject) => { this.pubSubConnection.sendCommands(authCommand, (error, results) => { @@ -96,7 +95,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { } public unsubscribe(channelName: string): void { - this.queryConnection.sql`UNLISTEN ${channelName};` + this.pubSubConnection.sql`UNLISTEN ${channelName};` } /** @@ -106,7 +105,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { */ public async create(channelName: string, failIfExists: boolean = true): Promise { // type this output - return await this.queryConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName) + return await this.pubSubConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName) } /** @@ -115,7 +114,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { */ public async delete(channelName: string): Promise { // type this output - return await this.queryConnection.sql(`REMOVE CHANNEL ?;`, channelName) + return await this.pubSubConnection.sql(`REMOVE CHANNEL ?;`, channelName) } /** @@ -123,7 +122,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { */ public notify(channelName: string, message: string): Promise { // type this output - return this.queryConnection.sql`NOTIFY ${channelName} ${message};` + return this.pubSubConnection.sql`NOTIFY ${channelName} ${message};` } /** @@ -137,7 +136,7 @@ export class SQLiteCloudPubSubClient implements SQLiteCloudPubSub { if (error) { reject(error) } else { - this.queryConnection.close() + this.pubSubConnection.close() resolve(results) } }) diff --git a/src/packages/storage/SQLiteCloudFileClient.ts b/src/packages/storage/SQLiteCloudStorageClient.ts similarity index 90% rename from src/packages/storage/SQLiteCloudFileClient.ts rename to src/packages/storage/SQLiteCloudStorageClient.ts index fd63505..033eb0c 100644 --- a/src/packages/storage/SQLiteCloudFileClient.ts +++ b/src/packages/storage/SQLiteCloudStorageClient.ts @@ -2,20 +2,18 @@ import { SQLiteCloudError } from "../../drivers/types" import { getAPIUrl } from "../utils" import { Fetch, fetchWithAuth } from "../utils/fetch" -interface SQLiteCloudFile { - createBucket(bucket: string, path: string): Promise +interface Storage { + createBucket(bucket: string): Promise getBucket(bucket: string): Promise - deleteBucket(bucket: string): Promise + deleteBucket(bucket: string): Promise listBuckets(): Promise - upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise + upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise download(bucket: string, pathname: string): Promise - remove(bucket: string, pathName: string): Promise + remove(bucket: string, pathName: string): Promise list(bucket: string): Promise } -const FILES_DATABASE = 'files.sqlite' - -export class SQLiteCloudFileClient implements SQLiteCloudFile { +export class StorageClient implements Storage { private filesUrl: string private webliteSQLUrl: string private fetch: Fetch @@ -102,4 +100,3 @@ export class SQLiteCloudFileClient implements SQLiteCloudFile { return response.json() } } - diff --git a/src/packages/utils/index.ts b/src/packages/utils/index.ts index 3b976ab..4d3e45f 100644 --- a/src/packages/utils/index.ts +++ b/src/packages/utils/index.ts @@ -23,3 +23,32 @@ export const getDefaultDatabase = (connectionString: string) => { export const getDbFromConfig = (config: SQLiteCloudConfig) => new URL(config.connectionstring ?? '')?.pathname.split('/').pop() ?? '' export const formatCommand = (arr: string[]) => arr.reduce((acc, curr) => curr.length > 0 ? (acc + ' ' + curr) : acc, '') + ';' + +/** + * Cleans and validates the SQLite Cloud connection string + * @param connectionString - The connection string to clean + * @returns The cleaned connection string + * @throws Error if connection string is invalid + * + * @example + * ```ts + * // Valid connection string + * cleanConnectionString('sqlitecloud://username:password@host:port/database') + * + * // Removes trailing slash + * cleanConnectionString('sqlitecloud://username:password@host:port/database/') + * + * // Throws error + * cleanConnectionString('invalid-connection-string') + * ``` + */ + +export const cleanConnectionString = (connectionString: string) => { + if (!connectionString.includes('sqlitecloud://')) { + throw new Error('Invalid connection string') + } + if (connectionString.endsWith('/')) { + return connectionString.slice(0, -1) + } + return connectionString +} \ No newline at end of file diff --git a/src/packages/weblite/SQLiteCloudWebliteClient.ts b/src/packages/weblite/SQLiteCloudWebliteClient.ts index 4a163d0..30eed0c 100644 --- a/src/packages/weblite/SQLiteCloudWebliteClient.ts +++ b/src/packages/weblite/SQLiteCloudWebliteClient.ts @@ -3,12 +3,16 @@ import { Fetch, fetchWithAuth } from '../utils/fetch' import { DEFAULT_HEADERS } from '../../drivers/constants' import { getAPIUrl } from '../utils' +interface WebliteResponse { + data: any, + error: SQLiteCloudError | null +} interface SQLiteCloudWeblite { - upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise - download(databaseName: string): Promise - delete(databaseName: string): Promise - listDatabases(): Promise - create(databaseName: string): Promise + upload(dbName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise + download(dbName: string): Promise + delete(dbName: string): Promise + listDatabases(): Promise + create(dbName: string): Promise } export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { @@ -20,70 +24,79 @@ export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { this.fetch = fetch || fetchWithAuth(connectionString) } - async upload(databaseName: string, file: File | Buffer | Blob | string, opts: UploadOptions = {}) { - const url = `${this.webliteUrl}/${databaseName}` - let body - if (file instanceof File) { - body = file - } else if (file instanceof Buffer) { - body = file - } else if (file instanceof Blob) { - body = file - } else { - // string - body = new Blob([file]) - } - - const headers = { - 'Content-Type': 'application/octet-stream', - 'X-Client-Info': DEFAULT_HEADERS['X-Client-Info'] - } - - const method = opts.replace ? 'PATCH' : 'POST' + async upload( + dbName: string, + file: File | Buffer | Blob | string, + opts: UploadOptions = {} + ) { + const url = `${this.webliteUrl}/${dbName}` + let body: File | Buffer | Blob | string + let headers = {} + if (file instanceof File) { + body = file - const response = await this.fetch(url, { method, body, headers }) + } else if (file instanceof Buffer) { + body = file + } else if (file instanceof Blob) { + body = file + } else { + // string + body = new Blob([file]) + } + + headers = { + ...(opts.headers ?? {}), + ...headers, + ...DEFAULT_HEADERS, + } + + const method = opts.replace ? 'PATCH' : 'POST' + const response = await this.fetch(url, { method, body, headers }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) } } - return response + const data = await response.json() + + return { data, error: null } } - async download(databaseName: string) { - const url = `${this.webliteUrl}/${databaseName}` + async download(dbName: string) { + const url = `${this.webliteUrl}/${dbName}` const response = await this.fetch(url, { method: 'GET' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to download database: ${response.statusText}`) } } const isNode = typeof window === 'undefined' - return isNode ? await response.arrayBuffer() : await response.blob() + const data = isNode ? await response.arrayBuffer() : await response.blob() + return { data, error: null } } - async delete(databaseName: string) { - const url = `${this.webliteUrl}/${databaseName}` + async delete(dbName: string) { + const url = `${this.webliteUrl}/${dbName}` const response = await this.fetch(url, { method: 'DELETE' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) } } - return response + return { data: null, error: null } } async listDatabases() { const url = `${this.webliteUrl}/databases` const response = await this.fetch(url, { method: 'GET' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) } } - return await response.json() + return { data: await response.json(), error: null } } - async create(databaseName: string) { - const response = await fetch(`${this.webliteUrl}/sql?sql=CREATE DATABASE ${databaseName}`, { method: 'POST' }) + async create(dbName: string) { + const response = await fetch(`${this.webliteUrl}/sql?sql=CREATE DATABASE ${dbName}`, { method: 'POST' }) if (!response.ok) { - throw new SQLiteCloudError(`Failed to create database: ${response.statusText}`) + return { data: null, error: new SQLiteCloudError(`Failed to create database: ${response.statusText}`) } } - return response + return { data: null, error: null } } } diff --git a/src/packages/test/SQLiteCloudClient.test.ts b/test/SQLiteCloudClient.test.ts similarity index 100% rename from src/packages/test/SQLiteCloudClient.test.ts rename to test/SQLiteCloudClient.test.ts From d3904cdd473193d72cb2465ce79019e718631e6c Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Tue, 24 Dec 2024 23:36:14 -0800 Subject: [PATCH 09/18] improve client library, apis and error handling --- src/packages/SQLiteCloudClient.ts | 8 +- src/packages/functions/FunctionsClient.ts | 61 +++++- src/packages/pubsub/PubSubClient.ts | 2 + .../storage/SQLiteCloudStorageClient.ts | 102 ---------- src/packages/storage/StorageClient.ts | 175 ++++++++++++++++++ .../vector/SQLiteCloudVectorClient.ts | 9 +- .../weblite/SQLiteCloudWebliteClient.ts | 22 ++- 7 files changed, 255 insertions(+), 124 deletions(-) delete mode 100644 src/packages/storage/SQLiteCloudStorageClient.ts create mode 100644 src/packages/storage/StorageClient.ts diff --git a/src/packages/SQLiteCloudClient.ts b/src/packages/SQLiteCloudClient.ts index 1c8c91d..3ed35bf 100644 --- a/src/packages/SQLiteCloudClient.ts +++ b/src/packages/SQLiteCloudClient.ts @@ -1,8 +1,8 @@ import { Database } from '../drivers/database' import { Fetch, fetchWithAuth } from './utils/fetch' import { PubSubClient } from './pubsub/PubSubClient' -import { SQLiteCloudWebliteClient } from './weblite/SQLiteCloudWebliteClient' -import { StorageClient } from './storage/SQLiteCloudStorageClient' +import { WebliteClient } from './weblite/SQLiteCloudWebliteClient' +import { StorageClient } from './storage/StorageClient' import { SQLiteCloudCommand, SQLiteCloudError } from '../drivers/types' import { cleanConnectionString, getDefaultDatabase } from './utils' @@ -59,11 +59,11 @@ export class SQLiteCloudClient { } get weblite() { - return new SQLiteCloudWebliteClient(this.connectionString, this.fetch) + return new WebliteClient(this.connectionString, { customFetch: this.fetch }) } get files() { - return new StorageClient(this.connectionString, this.fetch) + return new StorageClient(this.connectionString, { customFetch: this.fetch }) } get functions() { diff --git a/src/packages/functions/FunctionsClient.ts b/src/packages/functions/FunctionsClient.ts index 973e2b3..486e800 100644 --- a/src/packages/functions/FunctionsClient.ts +++ b/src/packages/functions/FunctionsClient.ts @@ -1,3 +1,4 @@ +import { DEFAULT_HEADERS } from '../../drivers/constants' import { SQLiteCloudError } from '../../drivers/types' import { FUNCTIONS_ROOT_PATH } from '../constants' import { getAPIUrl } from '../utils' @@ -17,20 +18,68 @@ export class FunctionsClient { ) { this.url = getAPIUrl(connectionString, FUNCTIONS_ROOT_PATH) this.fetch = resolveFetch(options.customFetch) - this.headers = options.headers ?? {} + this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } } // auth token is the full connection string with apikey setAuth(token: string) { this.headers.Authorization = `Bearer ${token}` } - async invoke(functionName: string, args: any[]) { + async invoke(functionId: string, args: any[]) { try { - // TODO IMPLEMENT + const response = await this.fetch(`${this.url}/${functionId}`, { + method: 'POST', + body: JSON.stringify(args), + headers: this.headers + }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to invoke function: ${response.statusText}`) + } + let responseType = (response.headers.get('Content-Type') ?? 'text/plain').split(';')[0].trim() + let data: any + if (responseType === 'application/json') { + data = await response.json() + } else if (responseType === 'application/octet-stream') { + data = await response.blob() + } else if (responseType === 'text/event-stream') { + data = response + } else if (responseType === 'multipart/form-data') { + data = await response.formData() + } else { + // default to text + data = await response.text() + } + return { data, error: null } } catch (error) { - throw new SQLiteCloudError(`Failed to invoke function: ${error}`) + return { data: null, error } } } - - } + +/** + if ( + functionArgs && + ((headers && !Object.prototype.hasOwnProperty.call(headers, 'Content-Type')) || !headers) + ) { + if ( + (typeof Blob !== 'undefined' && functionArgs instanceof Blob) || + functionArgs instanceof ArrayBuffer + ) { + // will work for File as File inherits Blob + // also works for ArrayBuffer as it is the same underlying structure as a Blob + _headers['Content-Type'] = 'application/octet-stream' + body = functionArgs + } else if (typeof functionArgs === 'string') { + // plain string + _headers['Content-Type'] = 'text/plain' + body = functionArgs + } else if (typeof FormData !== 'undefined' && functionArgs instanceof FormData) { + // don't set content-type headers + // Request will automatically add the right boundary value + body = functionArgs + } else { + // default, assume this is JSON + _headers['Content-Type'] = 'application/json' + body = JSON.stringify(functionArgs) + } + */ \ No newline at end of file diff --git a/src/packages/pubsub/PubSubClient.ts b/src/packages/pubsub/PubSubClient.ts index e8cdda7..500e88e 100644 --- a/src/packages/pubsub/PubSubClient.ts +++ b/src/packages/pubsub/PubSubClient.ts @@ -125,6 +125,8 @@ export class PubSubClient implements PubSub { return this.pubSubConnection.sql`NOTIFY ${channelName} ${message};` } + // DOUBLE CHECK THIS + /** * Ask the server to close the connection to the database and * to keep only open the Pub/Sub connection. diff --git a/src/packages/storage/SQLiteCloudStorageClient.ts b/src/packages/storage/SQLiteCloudStorageClient.ts deleted file mode 100644 index 033eb0c..0000000 --- a/src/packages/storage/SQLiteCloudStorageClient.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { SQLiteCloudError } from "../../drivers/types" -import { getAPIUrl } from "../utils" -import { Fetch, fetchWithAuth } from "../utils/fetch" - -interface Storage { - createBucket(bucket: string): Promise - getBucket(bucket: string): Promise - deleteBucket(bucket: string): Promise - listBuckets(): Promise - upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise - download(bucket: string, pathname: string): Promise - remove(bucket: string, pathName: string): Promise - list(bucket: string): Promise -} - -export class StorageClient implements Storage { - private filesUrl: string - private webliteSQLUrl: string - private fetch: Fetch - - constructor(connectionString: string, sql?: Fetch) { - this.filesUrl = getAPIUrl(connectionString, 'files') - this.webliteSQLUrl = getAPIUrl(connectionString, 'weblite/sql') - this.fetch = fetchWithAuth(connectionString, fetch) - } - - async createBucket(bucket: string) { - const url = `${this.webliteSQLUrl}?sql=USE DATABASE files; INSERT INTO files (Bucket) VALUES ('${bucket}');` - const response = await this.fetch(url, { method: 'POST' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to create bucket: ${response.statusText}`) - } - return response.json() - } - - async getBucket(bucket: string) { - const url = `${this.filesUrl}/${bucket}` - const response = await this.fetch(url, { method: 'GET' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to get bucket: ${response.statusText}`) - } - - return response.json() - } - - async deleteBucket(bucket: string) { - const url = `${this.filesUrl}/${bucket}` - const response = await this.fetch(url, { method: 'DELETE' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to delete bucket: ${response.statusText}`) - } - return response.json() - } - - async listBuckets() { - const encodedUrl = encodeURIComponent(`${this.webliteSQLUrl}?sql=USE DATABASE files.sqlite; SELECT * FROM files`) - const response = await this.fetch(encodedUrl) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to list buckets: ${response.statusText}`) - } - return response.json() - } - - async upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }) { - const url = `${this.filesUrl}/${bucket}/${pathname}`; - const headers = { - 'Content-Type': options?.contentType || 'application/octet-stream' - } - const response = await this.fetch(url, { method: 'POST', body: file, headers }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to upload file: ${response.statusText}`) - } - return response.json() - } - - async download(bucket: string, pathname: string) { - const url = `${this.filesUrl}/${bucket}/${pathname}`; - const response = await this.fetch(url, { method: 'GET' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to download file: ${response.statusText}`) - } - return response.blob() - } - - async remove(bucket: string, pathName: string) { - const url = `${this.filesUrl}/${bucket}/${pathName}` - const response = await this.fetch(url, { method: 'DELETE' }) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to remove file: ${response.statusText}`) - } - return response.json() - } - - async list(bucket: string) { - const encodedUrl = encodeURIComponent(`${this.webliteSQLUrl}?sql=USE DATABASE files.sqlite; SELECT * FROM files WHERE bucket = '${bucket}'`) - const response = await this.fetch(encodedUrl) - if (!response.ok) { - throw new SQLiteCloudError(`Failed to list files: ${response.statusText}`) - } - return response.json() - } -} diff --git a/src/packages/storage/StorageClient.ts b/src/packages/storage/StorageClient.ts new file mode 100644 index 0000000..bb6156b --- /dev/null +++ b/src/packages/storage/StorageClient.ts @@ -0,0 +1,175 @@ +import { DEFAULT_HEADERS } from "../../drivers/constants" +import { SQLiteCloudError } from "../../drivers/types" +import { getAPIUrl } from "../utils" +import { Fetch, fetchWithAuth } from "../utils/fetch" + +interface StorageResponse { + data: any + error: any +} + +interface Storage { + createBucket(bucket: string): Promise + getBucket(bucket: string): Promise + deleteBucket(bucket: string): Promise + listBuckets(): Promise + upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise + download(bucket: string, pathname: string): Promise + remove(bucket: string, pathName: string): Promise + list(bucket: string): Promise +} + +export class StorageClient implements Storage { + protected filesUrl: string + protected webliteSQLUrl: string + protected headers: Record + protected fetch: Fetch + + constructor( + connectionString: string, + options: { + customFetch?: Fetch, + headers?: Record + } = {}) { + this.filesUrl = getAPIUrl(connectionString, 'files') + this.webliteSQLUrl = getAPIUrl(connectionString, 'weblite/sql') + this.fetch = options.customFetch || fetchWithAuth(connectionString) + this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } + } + + async createBucket(bucket: string) { + const sql = `USE DATABASE files; INSERT INTO files (Bucket) VALUES ('${bucket}');` + + try { + const response = await this.fetch(this.webliteSQLUrl, { + method: 'POST', + body: JSON.stringify({ sql }), + headers: this.headers + }) + + if (!response.ok) { + throw new SQLiteCloudError(`Failed to create bucket: ${response.statusText}`) + } + + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async getBucket(bucket: string) { + const url = `${this.filesUrl}/${bucket}` + const response = await this.fetch(url, { method: 'GET', headers: this.headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to get bucket: ${response.statusText}`) + } + + return { data: await response.json(), error: null } + } + + async deleteBucket(bucket: string) { + const url = `${this.filesUrl}/${bucket}` + try { + const response = await this.fetch(url, { method: 'DELETE', headers: this.headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to delete bucket: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async listBuckets() { + const sql = `USE DATABASE files.sqlite; SELECT * FROM files;` + try { + const response = await this.fetch(this.webliteSQLUrl, { + method: 'POST', + body: JSON.stringify({ sql }), + headers: this.headers + }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list buckets: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { + data: null, + error + } + } + } + + async upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }) { + const url = `${this.filesUrl}/${bucket}/${pathname}`; + const headers = { + 'Content-Type': options?.contentType || 'application/octet-stream' + } + try { + const response = await this.fetch(url, { method: 'POST', body: file, headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to upload file: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async download(bucket: string, pathname: string) { + const url = `${this.filesUrl}/${bucket}/${pathname}`; + try { + const response = await this.fetch(url, { method: 'GET' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to download file: ${response.statusText}`) + } + let responseType = (response.headers.get('Content-Type') ?? 'text/plain').split(';')[0].trim() + let data: any + if (responseType === 'application/json') { + data = await response.json() + } else if (responseType === 'application/octet-stream') { + data = await response.blob() + } else if (responseType === 'text/event-stream') { + data = response + } else if (responseType === 'multipart/form-data') { + data = await response.formData() + } else { + // default to text + data = await response.text() + } + return { data, error: null } + } catch (error) { + return { data: null, error } + } + } + + async remove(bucket: string, pathName: string) { + const url = `${this.filesUrl}/${bucket}/${pathName}` + try { + const response = await this.fetch(url, { method: 'DELETE' }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to remove file: ${response.statusText}`) + } + return { data: response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } + + async list(bucket: string) { + const sql = `USE DATABASE files.sqlite; SELECT * FROM files WHERE bucket = '${bucket}'` + try { + const response = await this.fetch(this.webliteSQLUrl, { + method: 'POST', + body: JSON.stringify({ sql }), + headers: this.headers + }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list files: ${response.statusText}`) + } + return { data: await response.json(), error: null } + } catch (error) { + return { data: null, error } + } + } +} diff --git a/src/packages/vector/SQLiteCloudVectorClient.ts b/src/packages/vector/SQLiteCloudVectorClient.ts index d56f80e..449d9b2 100644 --- a/src/packages/vector/SQLiteCloudVectorClient.ts +++ b/src/packages/vector/SQLiteCloudVectorClient.ts @@ -22,9 +22,9 @@ interface QueryOptions { where?: string[] } -interface SQLiteCloudVector { - init(options: IndexOptions): Promise - upsert(data: UpsertData): Promise +interface Vector { + init(options: IndexOptions): Promise + upsert(data: UpsertData): Promise query(queryEmbedding: number[], options: QueryOptions): Promise } @@ -52,8 +52,7 @@ function formatUpsertCommand(data: UpsertData): [any, any] { } -export class SQLiteCloudVectorClient implements SQLiteCloudVector { - +export class VectorClient implements Vector { private _db: Database private _tableName: string private _columns: Column[] diff --git a/src/packages/weblite/SQLiteCloudWebliteClient.ts b/src/packages/weblite/SQLiteCloudWebliteClient.ts index 30eed0c..0e8a0bb 100644 --- a/src/packages/weblite/SQLiteCloudWebliteClient.ts +++ b/src/packages/weblite/SQLiteCloudWebliteClient.ts @@ -4,10 +4,10 @@ import { DEFAULT_HEADERS } from '../../drivers/constants' import { getAPIUrl } from '../utils' interface WebliteResponse { - data: any, + data: any, // TODO: type this error: SQLiteCloudError | null } -interface SQLiteCloudWeblite { +interface Weblite { upload(dbName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise download(dbName: string): Promise delete(dbName: string): Promise @@ -15,13 +15,21 @@ interface SQLiteCloudWeblite { create(dbName: string): Promise } -export class SQLiteCloudWebliteClient implements SQLiteCloudWeblite { - private webliteUrl: string - private fetch: Fetch +export class WebliteClient implements Weblite { + protected webliteUrl: string + protected headers: Record + protected fetch: Fetch - constructor(connectionString: string, fetch?: Fetch) { + constructor( + connectionString: string, + options: { + customFetch?: Fetch, + headers?: Record + } = {} + ) { this.webliteUrl = getAPIUrl(connectionString, 'weblite') - this.fetch = fetch || fetchWithAuth(connectionString) + this.fetch = options?.customFetch || fetchWithAuth(connectionString) + this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } } async upload( From b394d62f299c0bc544dd8fc6d6196d42f74c4d4f Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Tue, 24 Dec 2024 23:39:09 -0800 Subject: [PATCH 10/18] clean up functions client --- src/packages/functions/FunctionsClient.ts | 29 +++-------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/src/packages/functions/FunctionsClient.ts b/src/packages/functions/FunctionsClient.ts index 486e800..64e3535 100644 --- a/src/packages/functions/FunctionsClient.ts +++ b/src/packages/functions/FunctionsClient.ts @@ -2,7 +2,7 @@ import { DEFAULT_HEADERS } from '../../drivers/constants' import { SQLiteCloudError } from '../../drivers/types' import { FUNCTIONS_ROOT_PATH } from '../constants' import { getAPIUrl } from '../utils' -import { Fetch, resolveFetch, resolveHeadersConstructor } from '../utils/fetch' +import { Fetch, resolveFetch } from '../utils/fetch' export class FunctionsClient { protected url: string @@ -26,6 +26,7 @@ export class FunctionsClient { } async invoke(functionId: string, args: any[]) { + // add argument handling try { const response = await this.fetch(`${this.url}/${functionId}`, { method: 'POST', @@ -57,29 +58,5 @@ export class FunctionsClient { } /** - if ( - functionArgs && - ((headers && !Object.prototype.hasOwnProperty.call(headers, 'Content-Type')) || !headers) - ) { - if ( - (typeof Blob !== 'undefined' && functionArgs instanceof Blob) || - functionArgs instanceof ArrayBuffer - ) { - // will work for File as File inherits Blob - // also works for ArrayBuffer as it is the same underlying structure as a Blob - _headers['Content-Type'] = 'application/octet-stream' - body = functionArgs - } else if (typeof functionArgs === 'string') { - // plain string - _headers['Content-Type'] = 'text/plain' - body = functionArgs - } else if (typeof FormData !== 'undefined' && functionArgs instanceof FormData) { - // don't set content-type headers - // Request will automatically add the right boundary value - body = functionArgs - } else { - // default, assume this is JSON - _headers['Content-Type'] = 'application/json' - body = JSON.stringify(functionArgs) - } + */ \ No newline at end of file From 4f3ba4ff4a6c85fac1d68eb4ed5ee3d186123749 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Wed, 25 Dec 2024 11:39:53 -0800 Subject: [PATCH 11/18] clean up functions and pub sub --- src/packages/functions/FunctionsClient.ts | 59 +++++- src/packages/pubsub/PubSubClient.ts | 48 ++++- .../vector/SQLiteCloudVectorClient.ts | 190 +++++++++--------- 3 files changed, 183 insertions(+), 114 deletions(-) diff --git a/src/packages/functions/FunctionsClient.ts b/src/packages/functions/FunctionsClient.ts index 64e3535..964207b 100644 --- a/src/packages/functions/FunctionsClient.ts +++ b/src/packages/functions/FunctionsClient.ts @@ -4,6 +4,21 @@ import { FUNCTIONS_ROOT_PATH } from '../constants' import { getAPIUrl } from '../utils' import { Fetch, resolveFetch } from '../utils/fetch' +/** + * FunctionInvokeOptions + * @param args - The arguments to pass to the function. + * @param headers - The headers to pass to the function. + */ +interface FunctionInvokeOptions { + args: any[] + headers?: Record +} + +/** + * FunctionsClient + * @param invoke - Invoke a function. + * @param setAuth - Set the authentication token. + */ export class FunctionsClient { protected url: string protected fetch: Fetch @@ -20,22 +35,51 @@ export class FunctionsClient { this.fetch = resolveFetch(options.customFetch) this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } } - // auth token is the full connection string with apikey + // TODO: check authorization and api key setup in Gateway setAuth(token: string) { this.headers.Authorization = `Bearer ${token}` } - async invoke(functionId: string, args: any[]) { - // add argument handling + async invoke(functionId: string, options: FunctionInvokeOptions) { + const { headers, args } = options + let body; + let _headers: Record = {} + if (args && + ((headers && !Object.prototype.hasOwnProperty.call(headers, 'Content-Type')) || !headers) + ) { + if ( + (typeof Blob !== 'undefined' && args instanceof Blob) || + args instanceof ArrayBuffer + ) { + // will work for File as File inherits Blob + // also works for ArrayBuffer as it is the same underlying structure as a Blob + _headers['Content-Type'] = 'application/octet-stream' + body = args + } else if (typeof args === 'string') { + // plain string + _headers['Content-Type'] = 'text/plain' + body = args + } else if (typeof FormData !== 'undefined' && args instanceof FormData) { + _headers['Content-Type'] = 'multipart/form-data' + body = args + } else { + // default, assume this is JSON + _headers['Content-Type'] = 'application/json' + body = JSON.stringify(args) + } + } + try { const response = await this.fetch(`${this.url}/${functionId}`, { method: 'POST', body: JSON.stringify(args), - headers: this.headers + headers: { ..._headers, ...this.headers, ...headers } }) + if (!response.ok) { throw new SQLiteCloudError(`Failed to invoke function: ${response.statusText}`) } + let responseType = (response.headers.get('Content-Type') ?? 'text/plain').split(';')[0].trim() let data: any if (responseType === 'application/json') { @@ -47,16 +91,11 @@ export class FunctionsClient { } else if (responseType === 'multipart/form-data') { data = await response.formData() } else { - // default to text data = await response.text() } - return { data, error: null } + return { ...data, error: null } } catch (error) { return { data: null, error } } } } - -/** - - */ \ No newline at end of file diff --git a/src/packages/pubsub/PubSubClient.ts b/src/packages/pubsub/PubSubClient.ts index 500e88e..1a275c6 100644 --- a/src/packages/pubsub/PubSubClient.ts +++ b/src/packages/pubsub/PubSubClient.ts @@ -2,13 +2,36 @@ import { SQLiteCloudConnection } from '../../drivers/connection' import SQLiteCloudTlsConnection from '../../drivers/connection-tls' import { SQLiteCloudConfig } from '../../drivers/types' +/** + * PubSubCallback + * @param error - The error that occurred. + * @param results - The results of the operation. + */ export type PubSubCallback = (error: Error | null, results?: T) => void +/** + * ListenOptions + * @param tableName - The name of the table to listen to. + * @param dbName - The name of the database to listen to. + */ export interface ListenOptions { tableName: string dbName?: string } +/** + * PubSub + * @param listen - Listen to a channel and start to receive messages to the provided callback. + * @param unlisten - Stop receive messages from a table or channel. + * @param subscribe - Subscribe to a channel. + * @param unsubscribe - Unsubscribe from a channel. + * @param create - Create a channel. + * @param delete - Delete a channel. + * @param notify - Send a message to a channel. + * @param setPubSubOnly - Set the connection to Pub/Sub only. + * @param connected - Check if the connection is open. + * @param close - Close the connection. + */ export interface PubSub { listen(options: ListenOptions, callback: PubSubCallback): Promise unlisten(options: ListenOptions): void @@ -29,7 +52,7 @@ export class PubSubClient implements PubSub { protected _pubSubConnection: SQLiteCloudConnection | null protected defaultDatabaseName: string protected config: SQLiteCloudConfig - // instantiate in createConnection? + constructor(config: SQLiteCloudConfig) { this.config = config this._pubSubConnection = null @@ -71,14 +94,18 @@ export class PubSubClient implements PubSub { } /** - * Stop receive messages from a table or channel. - * @param entityType One of TABLE or CHANNEL - * @param entityName Name of the table or the channel + * Unlisten to a table. + * @param options Options for the unlisten operation. */ public unlisten(options: ListenOptions): void { this.pubSubConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` } + /** + * Subscribe (listen) to a channel. + * @param channelName The name of the channel to subscribe to. + * @param callback Callback to be called when a message is received. + */ public async subscribe(channelName: string, callback: PubSubCallback): Promise { const authCommand: string = await this.pubSubConnection.sql`LISTEN ${channelName};` @@ -94,6 +121,10 @@ export class PubSubClient implements PubSub { }) } + /** + * Unsubscribe (unlisten) from a channel. + * @param channelName The name of the channel to unsubscribe from. + */ public unsubscribe(channelName: string): void { this.pubSubConnection.sql`UNLISTEN ${channelName};` } @@ -104,8 +135,9 @@ export class PubSubClient implements PubSub { * @param failIfExists Raise an error if the channel already exists */ public async create(channelName: string, failIfExists: boolean = true): Promise { - // type this output - return await this.pubSubConnection.sql(`CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName) + return await this.pubSubConnection.sql( + `CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName + ) } /** @@ -113,15 +145,13 @@ export class PubSubClient implements PubSub { * @param name Channel name */ public async delete(channelName: string): Promise { - // type this output - return await this.pubSubConnection.sql(`REMOVE CHANNEL ?;`, channelName) + return await this.pubSubConnection.sql`REMOVE CHANNEL ${channelName};` } /** * Send a message to the channel. */ public notify(channelName: string, message: string): Promise { - // type this output return this.pubSubConnection.sql`NOTIFY ${channelName} ${message};` } diff --git a/src/packages/vector/SQLiteCloudVectorClient.ts b/src/packages/vector/SQLiteCloudVectorClient.ts index 449d9b2..b77ad40 100644 --- a/src/packages/vector/SQLiteCloudVectorClient.ts +++ b/src/packages/vector/SQLiteCloudVectorClient.ts @@ -1,95 +1,95 @@ -import { Database } from "../../drivers/database"; - -interface Column { - name: string; - type: string; - partitionKey?: boolean; - primaryKey?: boolean; -} - -interface IndexOptions { - tableName: string; - dimensions: number; - columns: Column[]; - binaryQuantization?: boolean; - dbName?: string; -} - -type UpsertData = [Record & { id: string | number }][] - -interface QueryOptions { - topK: number, - where?: string[] -} - -interface Vector { - init(options: IndexOptions): Promise - upsert(data: UpsertData): Promise - query(queryEmbedding: number[], options: QueryOptions): Promise -} - -const DEFAULT_EMBEDDING_COLUMN_NAME = 'embedding' - -const buildEmbeddingType = (dimensions: number, binaryQuantization: boolean) => { - return `${binaryQuantization ? 'BIT' : 'FLOAT'}[${dimensions}]` -} - -const formatInitColumns = (opts: IndexOptions) => { - const { columns, dimensions, binaryQuantization } = opts - return columns.reduce((acc, column) => { - let _type = column.type.toLowerCase(); - const { name, primaryKey, partitionKey } = column - if (_type === 'embedding') { - _type = buildEmbeddingType(dimensions, !!binaryQuantization) - } - const formattedColumn = `${name} ${_type} ${primaryKey ? 'PRIMARY KEY' : ''}${partitionKey ? 'PARTITION KEY' : ''}` - return `${acc}, ${formattedColumn}` - }, '') -} - -function formatUpsertCommand(data: UpsertData): [any, any] { - throw new Error("Function not implemented."); -} - - -export class VectorClient implements Vector { - private _db: Database - private _tableName: string - private _columns: Column[] - private _formattedColumns: string - - constructor(_db: Database) { - this._db = _db - this._tableName = '' - this._columns = [] - this._formattedColumns = '' - } - - async init(options: IndexOptions) { - const formattedColumns = formatInitColumns(options) - this._tableName = options.tableName - this._columns = options?.columns || [] - this._formattedColumns = formattedColumns - const useDbCommand = options?.dbName ? `USE DATABASE ${options.dbName}; ` : '' - const hasTable = await this._db.sql`${useDbCommand}SELECT 1 FROM ${options.tableName} LIMIT 1;` - - if (hasTable.length === 0) { // TODO - VERIFY CHECK HAS TABLE - const query = `CREATE VIRTUAL TABLE ${options.tableName} USING vec0(${formattedColumns})` - await this._db.sql(query) - } - return this - } - - async upsert(data: UpsertData) { - const [formattedColumns, formattedValues] = formatUpsertCommand(data) - const query = `INSERT INTO ${this._tableName}(${formattedColumns}) VALUES (${formattedValues})` - return await this._db.sql(query) - } - - async query(queryEmbedding: number[], options: QueryOptions) { - const query = `SELECT * FROM ${this._tableName} WHERE ${DEFAULT_EMBEDDING_COLUMN_NAME} match ${JSON.stringify(queryEmbedding)} and k = ${options.topK} and ${(options?.where?.join(' and ') || '')}` - const result = await this._db.sql(query) - return { data: result, error: null } - } - -} +// import { Database } from "../../drivers/database"; + +// interface Column { +// name: string; +// type: string; +// partitionKey?: boolean; +// primaryKey?: boolean; +// } + +// interface IndexOptions { +// tableName: string; +// dimensions: number; +// columns: Column[]; +// binaryQuantization?: boolean; +// dbName?: string; +// } + +// type UpsertData = [Record & { id: string | number }][] + +// interface QueryOptions { +// topK: number, +// where?: string[] +// } + +// interface Vector { +// init(options: IndexOptions): Promise +// upsert(data: UpsertData): Promise +// query(queryEmbedding: number[], options: QueryOptions): Promise +// } + +// const DEFAULT_EMBEDDING_COLUMN_NAME = 'embedding' + +// const buildEmbeddingType = (dimensions: number, binaryQuantization: boolean) => { +// return `${binaryQuantization ? 'BIT' : 'FLOAT'}[${dimensions}]` +// } + +// const formatInitColumns = (opts: IndexOptions) => { +// const { columns, dimensions, binaryQuantization } = opts +// return columns.reduce((acc, column) => { +// let _type = column.type.toLowerCase(); +// const { name, primaryKey, partitionKey } = column +// if (_type === 'embedding') { +// _type = buildEmbeddingType(dimensions, !!binaryQuantization) +// } +// const formattedColumn = `${name} ${_type} ${primaryKey ? 'PRIMARY KEY' : ''}${partitionKey ? 'PARTITION KEY' : ''}` +// return `${acc}, ${formattedColumn}` +// }, '') +// } + +// function formatUpsertCommand(data: UpsertData): [any, any] { +// throw new Error("Function not implemented."); +// } + + +// export class VectorClient implements Vector { +// private _db: Database +// private _tableName: string +// private _columns: Column[] +// private _formattedColumns: string + +// constructor(_db: Database) { +// this._db = _db +// this._tableName = '' +// this._columns = [] +// this._formattedColumns = '' +// } + +// async init(options: IndexOptions) { +// const formattedColumns = formatInitColumns(options) +// this._tableName = options.tableName +// this._columns = options?.columns || [] +// this._formattedColumns = formattedColumns +// const useDbCommand = options?.dbName ? `USE DATABASE ${options.dbName}; ` : '' +// const hasTable = await this._db.sql`${useDbCommand}SELECT 1 FROM ${options.tableName} LIMIT 1;` + +// if (hasTable.length === 0) { // TODO - VERIFY CHECK HAS TABLE +// const query = `CREATE VIRTUAL TABLE ${options.tableName} USING vec0(${formattedColumns})` +// await this._db.sql(query) +// } +// return this +// } + +// async upsert(data: UpsertData) { +// const [formattedColumns, formattedValues] = formatUpsertCommand(data) +// const query = `INSERT INTO ${this._tableName}(${formattedColumns}) VALUES (${formattedValues})` +// return await this._db.sql(query) +// } + +// async query(queryEmbedding: number[], options: QueryOptions) { +// const query = `SELECT * FROM ${this._tableName} WHERE ${DEFAULT_EMBEDDING_COLUMN_NAME} match ${JSON.stringify(queryEmbedding)} and k = ${options.topK} and ${(options?.where?.join(' and ') || '')}` +// const result = await this._db.sql(query) +// return { data: result, error: null } +// } + +// } From 4e601b3face089e8ec2a60ec1a7f083cb9cf27fc Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Wed, 25 Dec 2024 12:10:47 -0800 Subject: [PATCH 12/18] clean up storageclient --- src/packages/storage/StorageClient.ts | 43 +++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/packages/storage/StorageClient.ts b/src/packages/storage/StorageClient.ts index bb6156b..76dfdfb 100644 --- a/src/packages/storage/StorageClient.ts +++ b/src/packages/storage/StorageClient.ts @@ -3,22 +3,40 @@ import { SQLiteCloudError } from "../../drivers/types" import { getAPIUrl } from "../utils" import { Fetch, fetchWithAuth } from "../utils/fetch" +// TODO: add consistent return types + + +/** + * StorageResponse + * @param data - The data returned from the operation. + * @param error - The error that occurred. + */ interface StorageResponse { data: any error: any } +/** + * Storage + * @param createBucket - Create a bucket. + * @param getBucket - Get a bucket. + * @param deleteBucket - Delete a bucket. + * @param listBuckets - List all buckets. + * @param upload - Upload a file. + * @param download - Download a file. + * @param remove - Remove a file. + * @param list - List all files in a bucket. + */ interface Storage { createBucket(bucket: string): Promise getBucket(bucket: string): Promise deleteBucket(bucket: string): Promise listBuckets(): Promise - upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }): Promise + upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { headers?: Record }): Promise download(bucket: string, pathname: string): Promise remove(bucket: string, pathName: string): Promise list(bucket: string): Promise } - export class StorageClient implements Storage { protected filesUrl: string protected webliteSQLUrl: string @@ -100,10 +118,24 @@ export class StorageClient implements Storage { } } - async upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { contentType: string }) { + async upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { headers?: Record }) { const url = `${this.filesUrl}/${bucket}/${pathname}`; + let _headers: Record = {} + if (file instanceof File) { + _headers['Content-Type'] = file.type + } else if (file instanceof Blob) { + _headers['Content-Type'] = file.type + } else if (file instanceof Buffer) { + _headers['Content-Type'] = 'application/octet-stream' + } else if (typeof file === 'string') { + _headers['Content-Type'] = 'text/plain' + } else { + _headers['Content-Type'] = 'application/json' + } const headers = { - 'Content-Type': options?.contentType || 'application/octet-stream' + ..._headers, + ...options.headers, + ...this.headers } try { const response = await this.fetch(url, { method: 'POST', body: file, headers }) @@ -125,12 +157,11 @@ export class StorageClient implements Storage { } let responseType = (response.headers.get('Content-Type') ?? 'text/plain').split(';')[0].trim() let data: any + // TODO: add appropriate headers based on response type in Gateway if (responseType === 'application/json') { data = await response.json() } else if (responseType === 'application/octet-stream') { data = await response.blob() - } else if (responseType === 'text/event-stream') { - data = response } else if (responseType === 'multipart/form-data') { data = await response.formData() } else { From 8038b4903f29f708a52ae149eb49d60495b20dfa Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Wed, 25 Dec 2024 18:26:27 -0800 Subject: [PATCH 13/18] add back getPubSub to database --- package.json | 2 +- src/drivers/database.ts | 20 +++++++ src/packages/SQLiteCloudClient.ts | 46 +++++++++++----- .../FunctionsClient.ts | 25 ++++----- .../VectorClient.ts} | 0 src/packages/storage/StorageClient.ts | 36 +----------- src/packages/test/storage.test.ts | 18 ++++++ src/packages/test/utils.ts | 0 src/packages/types/index.d.ts | 55 +++++++++++++++++++ ...CloudWebliteClient.ts => WebliteClient.ts} | 24 +++----- 10 files changed, 146 insertions(+), 80 deletions(-) rename src/packages/{functions => _functions}/FunctionsClient.ts (80%) rename src/packages/{vector/SQLiteCloudVectorClient.ts => _vector/VectorClient.ts} (100%) create mode 100644 src/packages/test/storage.test.ts create mode 100644 src/packages/test/utils.ts create mode 100644 src/packages/types/index.d.ts rename src/packages/weblite/{SQLiteCloudWebliteClient.ts => WebliteClient.ts} (83%) diff --git a/package.json b/package.json index f63491f..33bb6ff 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.359", + "version": "1.0.360", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/drivers/database.ts b/src/drivers/database.ts index e4033d4..c3485e9 100644 --- a/src/drivers/database.ts +++ b/src/drivers/database.ts @@ -15,6 +15,7 @@ import { ErrorCallback, ResultsCallback, RowCallback, RowsCallback } from './typ import EventEmitter from 'eventemitter3' import { isBrowser } from './utilities' import { Statement } from './statement' +import { PubSub } from './pubsub' // Uses eventemitter3 instead of node events for browser compatibility // https://github.com/primus/eventemitter3 @@ -478,4 +479,23 @@ export class Database extends EventEmitter { }) }) } + /** + * PubSub class provides a Pub/Sub real-time updates and notifications system to + * allow multiple applications to communicate with each other asynchronously. + * It allows applications to subscribe to tables and receive notifications whenever + * data changes in the database table. It also enables sending messages to anyone + * subscribed to a specific channel. + * @returns {PubSub} A PubSub object + */ + public async getPubSub(): Promise { + return new Promise((resolve, reject) => { + this.getConnection((error, connection) => { + if (error || !connection) { + reject(error) + } else { + resolve(new PubSub(connection)) + } + }) + }) + } } diff --git a/src/packages/SQLiteCloudClient.ts b/src/packages/SQLiteCloudClient.ts index 3ed35bf..f6ba3db 100644 --- a/src/packages/SQLiteCloudClient.ts +++ b/src/packages/SQLiteCloudClient.ts @@ -1,19 +1,18 @@ import { Database } from '../drivers/database' import { Fetch, fetchWithAuth } from './utils/fetch' import { PubSubClient } from './pubsub/PubSubClient' -import { WebliteClient } from './weblite/SQLiteCloudWebliteClient' +import { WebliteClient } from './weblite/WebliteClient' import { StorageClient } from './storage/StorageClient' import { SQLiteCloudCommand, SQLiteCloudError } from '../drivers/types' import { cleanConnectionString, getDefaultDatabase } from './utils' - -interface SQLiteCloudClientConfig { - connectionString: string - fetch?: Fetch -} +import { FunctionsClient } from './_functions/FunctionsClient' +import { SQLiteCloudClientConfig } from './types' export class SQLiteCloudClient { protected connectionString: string protected fetch: Fetch + protected globalHeaders: Record + protected _defaultDb: string protected _db: Database constructor(config: SQLiteCloudClientConfig | string) { @@ -23,25 +22,30 @@ export class SQLiteCloudClient { } let connectionString: string let customFetch: Fetch | undefined + let globalHeaders: Record = {} if (typeof config === 'string') { connectionString = cleanConnectionString(config) + globalHeaders = {} } else { connectionString = config.connectionString - customFetch = config.fetch + customFetch = config.global?.fetch + globalHeaders = config.global?.headers ?? {} } - + this.connectionString = connectionString this.fetch = fetchWithAuth(this.connectionString, customFetch) - this.defaultDb = getDefaultDatabase(this.connectionString) ?? '' + this.globalHeaders = globalHeaders + this._defaultDb = getDefaultDatabase(this.connectionString) ?? '' this._db = new Database(this.connectionString) + } catch (error) { throw new SQLiteCloudError('failed to initialize SQLiteCloudClient') } } async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { - this.db.exec(`USE DATABASE ${this.defaultDb}`) + this.db.exec(`USE DATABASE ${this._defaultDb}`) try { const result = await this.db.sql(sql, ...values) return { data: result, error: null } @@ -59,20 +63,32 @@ export class SQLiteCloudClient { } get weblite() { - return new WebliteClient(this.connectionString, { customFetch: this.fetch }) + return new WebliteClient(this.connectionString, { + customFetch: this.fetch, + headers: this.globalHeaders + }) } get files() { - return new StorageClient(this.connectionString, { customFetch: this.fetch }) + return new StorageClient(this.connectionString, { + customFetch: this.fetch, + headers: this.globalHeaders + }) } get functions() { - // return new SQLiteCloudFunctionsClient(this.connectionString, this.fetch) - return null + return new FunctionsClient(this.connectionString, { + customFetch: this.fetch, + headers: this.globalHeaders + }) } set defaultDb(dbName: string) { - this.defaultDb = dbName + this._defaultDb = dbName + } + + get defaultDb() { + return this._defaultDb } } diff --git a/src/packages/functions/FunctionsClient.ts b/src/packages/_functions/FunctionsClient.ts similarity index 80% rename from src/packages/functions/FunctionsClient.ts rename to src/packages/_functions/FunctionsClient.ts index 964207b..0b0a738 100644 --- a/src/packages/functions/FunctionsClient.ts +++ b/src/packages/_functions/FunctionsClient.ts @@ -41,39 +41,38 @@ export class FunctionsClient { } async invoke(functionId: string, options: FunctionInvokeOptions) { - const { headers, args } = options let body; let _headers: Record = {} - if (args && - ((headers && !Object.prototype.hasOwnProperty.call(headers, 'Content-Type')) || !headers) + if (options.args && + ((options.headers && !Object.prototype.hasOwnProperty.call(options.headers, 'Content-Type')) || !options.headers) ) { if ( - (typeof Blob !== 'undefined' && args instanceof Blob) || - args instanceof ArrayBuffer + (typeof Blob !== 'undefined' && options.args instanceof Blob) || + options.args instanceof ArrayBuffer ) { // will work for File as File inherits Blob // also works for ArrayBuffer as it is the same underlying structure as a Blob _headers['Content-Type'] = 'application/octet-stream' - body = args - } else if (typeof args === 'string') { + body = options.args + } else if (typeof options.args === 'string') { // plain string _headers['Content-Type'] = 'text/plain' - body = args - } else if (typeof FormData !== 'undefined' && args instanceof FormData) { + body = options.args + } else if (typeof FormData !== 'undefined' && options.args instanceof FormData) { _headers['Content-Type'] = 'multipart/form-data' - body = args + body = options.args } else { // default, assume this is JSON _headers['Content-Type'] = 'application/json' - body = JSON.stringify(args) + body = JSON.stringify(options.args) } } try { const response = await this.fetch(`${this.url}/${functionId}`, { method: 'POST', - body: JSON.stringify(args), - headers: { ..._headers, ...this.headers, ...headers } + body: JSON.stringify(options.args), + headers: { ..._headers, ...this.headers, ...options.headers } }) if (!response.ok) { diff --git a/src/packages/vector/SQLiteCloudVectorClient.ts b/src/packages/_vector/VectorClient.ts similarity index 100% rename from src/packages/vector/SQLiteCloudVectorClient.ts rename to src/packages/_vector/VectorClient.ts diff --git a/src/packages/storage/StorageClient.ts b/src/packages/storage/StorageClient.ts index 76dfdfb..8c6975b 100644 --- a/src/packages/storage/StorageClient.ts +++ b/src/packages/storage/StorageClient.ts @@ -2,41 +2,9 @@ import { DEFAULT_HEADERS } from "../../drivers/constants" import { SQLiteCloudError } from "../../drivers/types" import { getAPIUrl } from "../utils" import { Fetch, fetchWithAuth } from "../utils/fetch" +import { Storage } from "../types" // TODO: add consistent return types - - -/** - * StorageResponse - * @param data - The data returned from the operation. - * @param error - The error that occurred. - */ -interface StorageResponse { - data: any - error: any -} - -/** - * Storage - * @param createBucket - Create a bucket. - * @param getBucket - Get a bucket. - * @param deleteBucket - Delete a bucket. - * @param listBuckets - List all buckets. - * @param upload - Upload a file. - * @param download - Download a file. - * @param remove - Remove a file. - * @param list - List all files in a bucket. - */ -interface Storage { - createBucket(bucket: string): Promise - getBucket(bucket: string): Promise - deleteBucket(bucket: string): Promise - listBuckets(): Promise - upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { headers?: Record }): Promise - download(bucket: string, pathname: string): Promise - remove(bucket: string, pathName: string): Promise - list(bucket: string): Promise -} export class StorageClient implements Storage { protected filesUrl: string protected webliteSQLUrl: string @@ -187,7 +155,7 @@ export class StorageClient implements Storage { } } - async list(bucket: string) { + async listBucketContents(bucket: string) { const sql = `USE DATABASE files.sqlite; SELECT * FROM files WHERE bucket = '${bucket}'` try { const response = await this.fetch(this.webliteSQLUrl, { diff --git a/src/packages/test/storage.test.ts b/src/packages/test/storage.test.ts new file mode 100644 index 0000000..79ee6e1 --- /dev/null +++ b/src/packages/test/storage.test.ts @@ -0,0 +1,18 @@ +import { expect } from '@jest/globals' +import { StorageClient } from '../storage/StorageClient' +import { CHINOOK_DATABASE_URL } from '../../../test/shared' + + +const storage = new StorageClient(CHINOOK_DATABASE_URL) + +describe('StorageClient', () => { + it('should be able to create a bucket', async () => { + expect(storage).toBeDefined() + + const bucket = await storage.createBucket('test-bucket') + + expect(bucket).toBeDefined() + }) +}) + + diff --git a/src/packages/test/utils.ts b/src/packages/test/utils.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/packages/types/index.d.ts b/src/packages/types/index.d.ts new file mode 100644 index 0000000..27a99ad --- /dev/null +++ b/src/packages/types/index.d.ts @@ -0,0 +1,55 @@ +import { Fetch } from '../utils/fetch' + + +export interface SQLiteCloudClientConfig { + connectionString: string + global?: { + fetch?: Fetch + headers?: Record + } +} + +export interface WebliteResponse { + data: any, // TODO: type this + error: SQLiteCloudError | null +} +export interface Weblite { + upload(dbName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise + download(dbName: string): Promise + delete(dbName: string): Promise + listDatabases(): Promise + create(dbName: string): Promise +} + + +/** + * StorageResponse + * @param data - The data returned from the operation. + * @param error - The error that occurred. + */ +interface StorageResponse { + data: any + error: any +} + +/** + * Storage + * @param createBucket - Create a bucket. + * @param getBucket - Get a bucket. + * @param deleteBucket - Delete a bucket. + * @param listBuckets - List all buckets. + * @param upload - Upload a file. + * @param download - Download a file. + * @param remove - Remove a file. + * @param list - List all files in a bucket. + */ +interface Storage { + createBucket(bucket: string): Promise + getBucket(bucket: string): Promise + deleteBucket(bucket: string): Promise + listBuckets(): Promise + upload(bucket: string, pathname: string, file: File | Buffer | Blob | string, options: { headers?: Record }): Promise + download(bucket: string, pathname: string): Promise + remove(bucket: string, pathName: string): Promise + listBucketContents(bucket: string): Promise +} \ No newline at end of file diff --git a/src/packages/weblite/SQLiteCloudWebliteClient.ts b/src/packages/weblite/WebliteClient.ts similarity index 83% rename from src/packages/weblite/SQLiteCloudWebliteClient.ts rename to src/packages/weblite/WebliteClient.ts index 0e8a0bb..73f016e 100644 --- a/src/packages/weblite/SQLiteCloudWebliteClient.ts +++ b/src/packages/weblite/WebliteClient.ts @@ -2,18 +2,7 @@ import { SQLiteCloudError, UploadOptions } from '../../drivers/types' import { Fetch, fetchWithAuth } from '../utils/fetch' import { DEFAULT_HEADERS } from '../../drivers/constants' import { getAPIUrl } from '../utils' - -interface WebliteResponse { - data: any, // TODO: type this - error: SQLiteCloudError | null -} -interface Weblite { - upload(dbName: string, file: File | Buffer | Blob | string, opts: UploadOptions): Promise - download(dbName: string): Promise - delete(dbName: string): Promise - listDatabases(): Promise - create(dbName: string): Promise -} +import { Weblite } from '../types' export class WebliteClient implements Weblite { protected webliteUrl: string @@ -25,11 +14,13 @@ export class WebliteClient implements Weblite { options: { customFetch?: Fetch, headers?: Record - } = {} + } = { + headers: {} + } ) { this.webliteUrl = getAPIUrl(connectionString, 'weblite') this.fetch = options?.customFetch || fetchWithAuth(connectionString) - this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } + this.headers = { ...DEFAULT_HEADERS, ...options.headers } } async upload( @@ -39,7 +30,6 @@ export class WebliteClient implements Weblite { ) { const url = `${this.webliteUrl}/${dbName}` let body: File | Buffer | Blob | string - let headers = {} if (file instanceof File) { body = file @@ -52,9 +42,9 @@ export class WebliteClient implements Weblite { body = new Blob([file]) } - headers = { + const headers = { ...(opts.headers ?? {}), - ...headers, + ...this.headers, ...DEFAULT_HEADERS, } From 3c7386577690d22527471290ebb2857685903b26 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Thu, 26 Dec 2024 00:15:19 -0800 Subject: [PATCH 14/18] refactor weblite --- src/drivers/constants.ts | 1 - src/drivers/database.ts | 1 + src/packages/SQLiteCloudClient.ts | 88 +++++---- .../{pubsub => _pubsub}/PubSubClient.ts | 0 .../{storage => _storage}/StorageClient.ts | 22 ++- src/packages/test/client.test.ts | 27 +++ src/packages/test/storage.test.ts | 31 ++- src/packages/types/index.d.ts | 1 - src/packages/utils/fetch.ts | 2 +- src/packages/weblite/WebliteClient.ts | 177 ++++++++++++------ 10 files changed, 239 insertions(+), 111 deletions(-) rename src/packages/{pubsub => _pubsub}/PubSubClient.ts (100%) rename src/packages/{storage => _storage}/StorageClient.ts (92%) create mode 100644 src/packages/test/client.test.ts diff --git a/src/drivers/constants.ts b/src/drivers/constants.ts index fa6bea9..94c096d 100644 --- a/src/drivers/constants.ts +++ b/src/drivers/constants.ts @@ -13,7 +13,6 @@ if (typeof Deno !== 'undefined') { export const DEFAULT_HEADERS = { 'X-Client-Info': `sqlitecloud-js-${JS_ENV}/${version}`, - 'Content-Type': 'application/octet-stream' } export const DEFAULT_GLOBAL_OPTIONS = { headers: DEFAULT_HEADERS diff --git a/src/drivers/database.ts b/src/drivers/database.ts index c3485e9..16e5920 100644 --- a/src/drivers/database.ts +++ b/src/drivers/database.ts @@ -486,6 +486,7 @@ export class Database extends EventEmitter { * data changes in the database table. It also enables sending messages to anyone * subscribed to a specific channel. * @returns {PubSub} A PubSub object + * DEPRECATED: use PubSubClient instead */ public async getPubSub(): Promise { return new Promise((resolve, reject) => { diff --git a/src/packages/SQLiteCloudClient.ts b/src/packages/SQLiteCloudClient.ts index f6ba3db..6fa97ab 100644 --- a/src/packages/SQLiteCloudClient.ts +++ b/src/packages/SQLiteCloudClient.ts @@ -1,95 +1,103 @@ import { Database } from '../drivers/database' import { Fetch, fetchWithAuth } from './utils/fetch' -import { PubSubClient } from './pubsub/PubSubClient' +import { PubSubClient } from './_pubsub/PubSubClient' import { WebliteClient } from './weblite/WebliteClient' -import { StorageClient } from './storage/StorageClient' -import { SQLiteCloudCommand, SQLiteCloudError } from '../drivers/types' -import { cleanConnectionString, getDefaultDatabase } from './utils' +import { StorageClient } from './_storage/StorageClient' +import { SQLiteCloudDataTypes, SQLiteCloudError } from '../drivers/types' +import { cleanConnectionString } from './utils' import { FunctionsClient } from './_functions/FunctionsClient' import { SQLiteCloudClientConfig } from './types' +import { DEFAULT_HEADERS } from '../drivers/constants' + +const validateConfig = (config: SQLiteCloudClientConfig | string) => { + if (!(config)) throw new SQLiteCloudError('No configuration provided') + if (typeof config === 'string') { + if (!config.includes('sqlitecloud://')) throw new SQLiteCloudError('Invalid connection string') + } + + if (typeof config === 'object') { + if (!config.connectionString) throw new SQLiteCloudError('No connection string provided') + if (!config.connectionString.includes('sqlitecloud://')) throw new SQLiteCloudError('Invalid connection string') + } +} export class SQLiteCloudClient { protected connectionString: string protected fetch: Fetch - protected globalHeaders: Record - protected _defaultDb: string - protected _db: Database + protected _globalHeaders: Record + protected _db: Database | null + protected _pubSub: PubSubClient | null + protected _weblite: WebliteClient | null constructor(config: SQLiteCloudClientConfig | string) { try { - if (!config) { - throw new SQLiteCloudError('Invalid connection string or config') - } + validateConfig(config) let connectionString: string let customFetch: Fetch | undefined let globalHeaders: Record = {} - + if (typeof config === 'string') { connectionString = cleanConnectionString(config) - globalHeaders = {} + globalHeaders = DEFAULT_HEADERS } else { connectionString = config.connectionString customFetch = config.global?.fetch - globalHeaders = config.global?.headers ?? {} + globalHeaders = config.global?.headers ? { ...DEFAULT_HEADERS, ...config.global.headers } : DEFAULT_HEADERS } this.connectionString = connectionString this.fetch = fetchWithAuth(this.connectionString, customFetch) - this.globalHeaders = globalHeaders - this._defaultDb = getDefaultDatabase(this.connectionString) ?? '' - this._db = new Database(this.connectionString) + this._globalHeaders = globalHeaders + this._db = null + this._pubSub = null + this._weblite = null } catch (error) { throw new SQLiteCloudError('failed to initialize SQLiteCloudClient') } } - - async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]) { - this.db.exec(`USE DATABASE ${this._defaultDb}`) - try { - const result = await this.db.sql(sql, ...values) - return { data: result, error: null } - } catch (error) { - return { error, data: null } - } + // Defaults to HTTP API + async sql(sql: TemplateStringsArray | string, ...values: SQLiteCloudDataTypes[]) { + return await this.weblite.sql(sql, ...values) } get pubSub() { - return new PubSubClient(this.db.getConfiguration()) + if (!this._pubSub) { + this._pubSub = new PubSubClient(this.db.getConfiguration()) + } + return this._pubSub } get db() { + if (!this._db) { + this._db = new Database(this.connectionString) + } return this._db } get weblite() { - return new WebliteClient(this.connectionString, { - customFetch: this.fetch, - headers: this.globalHeaders - }) + if (!this._weblite) { + this._weblite = new WebliteClient(this.connectionString, { + fetch: this.fetch, + headers: this._globalHeaders + }) + } + return this._weblite } get files() { return new StorageClient(this.connectionString, { customFetch: this.fetch, - headers: this.globalHeaders + headers: this._globalHeaders }) } get functions() { return new FunctionsClient(this.connectionString, { customFetch: this.fetch, - headers: this.globalHeaders + headers: this._globalHeaders }) } - - set defaultDb(dbName: string) { - this._defaultDb = dbName - } - - get defaultDb() { - return this._defaultDb - } } export function createClient(config: SQLiteCloudClientConfig | string): SQLiteCloudClient { diff --git a/src/packages/pubsub/PubSubClient.ts b/src/packages/_pubsub/PubSubClient.ts similarity index 100% rename from src/packages/pubsub/PubSubClient.ts rename to src/packages/_pubsub/PubSubClient.ts diff --git a/src/packages/storage/StorageClient.ts b/src/packages/_storage/StorageClient.ts similarity index 92% rename from src/packages/storage/StorageClient.ts rename to src/packages/_storage/StorageClient.ts index 8c6975b..a1dfe09 100644 --- a/src/packages/storage/StorageClient.ts +++ b/src/packages/_storage/StorageClient.ts @@ -5,6 +5,7 @@ import { Fetch, fetchWithAuth } from "../utils/fetch" import { Storage } from "../types" // TODO: add consistent return types + export class StorageClient implements Storage { protected filesUrl: string protected webliteSQLUrl: string @@ -16,30 +17,33 @@ export class StorageClient implements Storage { options: { customFetch?: Fetch, headers?: Record - } = {}) { + } = { + headers: {} + }) { this.filesUrl = getAPIUrl(connectionString, 'files') this.webliteSQLUrl = getAPIUrl(connectionString, 'weblite/sql') this.fetch = options.customFetch || fetchWithAuth(connectionString) - this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } + this.headers = { ...DEFAULT_HEADERS, ...options.headers } } async createBucket(bucket: string) { - const sql = `USE DATABASE files; INSERT INTO files (Bucket) VALUES ('${bucket}');` - try { const response = await this.fetch(this.webliteSQLUrl, { - method: 'POST', - body: JSON.stringify({ sql }), - headers: this.headers + method: 'POST', + body: JSON.stringify({ + database: 'files.sqlite', + sql: `INSERT INTO files (Bucket, Pathname, Data) VALUES ('${bucket}', '/', '' );` } + ), + headers: this.headers, }) if (!response.ok) { throw new SQLiteCloudError(`Failed to create bucket: ${response.statusText}`) } - return { data: await response.json(), error: null } + return await response.json(); } catch (error) { - return { data: null, error } + return { error, data: null, metadata: null } } } diff --git a/src/packages/test/client.test.ts b/src/packages/test/client.test.ts new file mode 100644 index 0000000..817e58f --- /dev/null +++ b/src/packages/test/client.test.ts @@ -0,0 +1,27 @@ +import { CHINOOK_DATABASE_URL } from '../../../test/shared' +import { SQLiteCloudClient } from '../SQLiteCloudClient' + +const DEFAULT_TABLE_NAME = 'albums'; + +const client = new SQLiteCloudClient(CHINOOK_DATABASE_URL) + +describe('SQLiteCloudClient test suite', () => { + it('should be able to create a client', () => { + expect(client).toBeDefined() + expect(client).toBeInstanceOf(SQLiteCloudClient) + }) + + it('should throw errors if no valid params are provided', () => { + expect(() => new SQLiteCloudClient('')).toThrow() + expect(() => new SQLiteCloudClient({ connectionString: '' })).toThrow() + expect(() => new SQLiteCloudClient({ connectionString: 'invalid' })).toThrow() + }) + + it('should be able to query the database', async () => { + const { data, error } = await client.sql`SELECT * FROM ${DEFAULT_TABLE_NAME}`; + + expect(data).toBeDefined() + expect(error).toBeNull() + }) + +}) diff --git a/src/packages/test/storage.test.ts b/src/packages/test/storage.test.ts index 79ee6e1..dbf38bc 100644 --- a/src/packages/test/storage.test.ts +++ b/src/packages/test/storage.test.ts @@ -1,18 +1,37 @@ import { expect } from '@jest/globals' -import { StorageClient } from '../storage/StorageClient' +import { StorageClient } from '../_storage/StorageClient' import { CHINOOK_DATABASE_URL } from '../../../test/shared' +const TEST_BUCKET_NAME = 'test_bucket' -const storage = new StorageClient(CHINOOK_DATABASE_URL) +const storage = new StorageClient(CHINOOK_DATABASE_URL, + { + headers: { + 'Content-Type': 'application/json' + } + } +) describe('StorageClient', () => { it('should be able to create a bucket', async () => { expect(storage).toBeDefined() + const getBucketResponse = await storage.getBucket(TEST_BUCKET_NAME) + console.log(getBucketResponse) - const bucket = await storage.createBucket('test-bucket') - - expect(bucket).toBeDefined() + const { data, error } = await storage.createBucket(TEST_BUCKET_NAME) + console.log(data, error) + expect(error).toBeNull() + expect(data).toBeDefined() }) -}) + it('should get a bucket', async () => { + expect(storage).toBeDefined() + + const { data, error } = await storage.getBucket(TEST_BUCKET_NAME) + console.log(data) + expect(error).toBeNull() + expect(data).toBeDefined() + }) + +}) diff --git a/src/packages/types/index.d.ts b/src/packages/types/index.d.ts index 27a99ad..3bc59ff 100644 --- a/src/packages/types/index.d.ts +++ b/src/packages/types/index.d.ts @@ -44,7 +44,6 @@ interface StorageResponse { * @param list - List all files in a bucket. */ interface Storage { - createBucket(bucket: string): Promise getBucket(bucket: string): Promise deleteBucket(bucket: string): Promise listBuckets(): Promise diff --git a/src/packages/utils/fetch.ts b/src/packages/utils/fetch.ts index d51dc7e..4a11340 100644 --- a/src/packages/utils/fetch.ts +++ b/src/packages/utils/fetch.ts @@ -27,7 +27,7 @@ export const fetchWithAuth = (authorization: string, customFetch?: Fetch): Fetch const fetch = resolveFetch(customFetch) const HeadersConstructor = resolveHeadersConstructor() - return async (input, init) => { + return (input, init) => { const headers = new HeadersConstructor(init?.headers) if (!headers.has('Authorization')) { headers.set('Authorization', `Bearer ${authorization}`) diff --git a/src/packages/weblite/WebliteClient.ts b/src/packages/weblite/WebliteClient.ts index 73f016e..933867f 100644 --- a/src/packages/weblite/WebliteClient.ts +++ b/src/packages/weblite/WebliteClient.ts @@ -1,45 +1,90 @@ -import { SQLiteCloudError, UploadOptions } from '../../drivers/types' +import { SQLiteCloudDataTypes, SQLiteCloudError, UploadOptions } from '../../drivers/types' import { Fetch, fetchWithAuth } from '../utils/fetch' import { DEFAULT_HEADERS } from '../../drivers/constants' -import { getAPIUrl } from '../utils' -import { Weblite } from '../types' +import { getAPIUrl, getDefaultDatabase } from '../utils' -export class WebliteClient implements Weblite { - protected webliteUrl: string +// Weblite Client - interact with SQLite Cloud via HTTP +export class WebliteClient { + protected baseUrl?: string // /weblite url protected headers: Record protected fetch: Fetch + protected _defaultDatabase?: string constructor( - connectionString: string, + connectionString: string, // sqlitecloud://xxx.xxx.xxx:port/database?apikey=xxx options: { - customFetch?: Fetch, + fetch?: Fetch, headers?: Record } = { headers: {} } ) { - this.webliteUrl = getAPIUrl(connectionString, 'weblite') - this.fetch = options?.customFetch || fetchWithAuth(connectionString) - this.headers = { ...DEFAULT_HEADERS, ...options.headers } + this.baseUrl = getAPIUrl(connectionString, 'weblite') + this.fetch = options?.fetch || fetchWithAuth(connectionString) + this.headers = { ...options.headers } + this._defaultDatabase = getDefaultDatabase(connectionString) } - async upload( - dbName: string, - file: File | Buffer | Blob | string, + async sql(sql: TemplateStringsArray | string, ...values: SQLiteCloudDataTypes[]) { + const url = `${this.baseUrl}/sql` + + try { + let _sql = '' + if (Array.isArray(sql) && 'raw' in sql) { // check raw property? + sql.forEach((string, i) => { + // TemplateStringsArray splits the string before each variable + // used in the template. Add the question mark + // to the end of the string for the number of used variables. + _sql += string + (i < values.length ? '?' : '') + }) + } else if (typeof sql === 'string') { + _sql = sql + } else { + throw new SQLiteCloudError('Invalid sql') + } + + const response = await this.fetch(url, { + method: 'POST', + body: JSON.stringify({ sql: _sql, bind: values }), + headers: { ...this.headers, 'Content-Type': 'application/json' } + }) + + if (!response.ok) { + throw new SQLiteCloudError(`Failed to execute sql: ${response.statusText}`) + } + return await response.json() + } catch (error) { + return { data: null, error } + } + } + + get defaultDatabase() { + return this._defaultDatabase + } + + useDatabase(name: string) { + this._defaultDatabase = name + return this + } + + async uploadDatabase( + filename: string, + database: File | Buffer | Blob | string, opts: UploadOptions = {} ) { - const url = `${this.webliteUrl}/${dbName}` - let body: File | Buffer | Blob | string - if (file instanceof File) { - body = file + const filenamePath = encodeURIComponent(filename) + const url = `${this.baseUrl}/${filenamePath}` - } else if (file instanceof Buffer) { - body = file - } else if (file instanceof Blob) { - body = file + let body: File | Buffer | Blob | string + if (database instanceof File) { + body = database + } else if (database instanceof Buffer) { + body = database + } else if (database instanceof Blob) { + body = database } else { // string - body = new Blob([file]) + body = new Blob([database]) } const headers = { @@ -49,52 +94,78 @@ export class WebliteClient implements Weblite { } const method = opts.replace ? 'PATCH' : 'POST' - const response = await this.fetch(url, { method, body, headers }) - if (!response.ok) { - return { data: null, error: new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) } + try { + const response = await this.fetch(url, { method, body, headers }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) + } + return await response.json() + } catch (error) { + return { data: null, error } } - - const data = await response.json() - - return { data, error: null } } - async download(dbName: string) { - const url = `${this.webliteUrl}/${dbName}` - const response = await this.fetch(url, { method: 'GET' }) - if (!response.ok) { - return { data: null, error: new SQLiteCloudError(`Failed to download database: ${response.statusText}`) } + async downloadDatabase( + filename: string, + ) { + const filenamePath = encodeURIComponent(filename) + const url = `${this.baseUrl}/${filenamePath}` + try { + const response = await this.fetch(url, { method: 'GET', headers: { ...this.headers } }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`) + } + const isNode = typeof window === 'undefined' + return isNode ? await response.arrayBuffer() : await response.blob() + } catch (error) { + return { data: null, error } } - const isNode = typeof window === 'undefined' - const data = isNode ? await response.arrayBuffer() : await response.blob() - return { data, error: null } } - async delete(dbName: string) { - const url = `${this.webliteUrl}/${dbName}` - const response = await this.fetch(url, { method: 'DELETE' }) - if (!response.ok) { - return { data: null, error: new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) } + async deleteDatabase(filename: string) { + const filenamePath = encodeURIComponent(filename) + const url = `${this.baseUrl}/${filenamePath}` + try { + const response = await this.fetch( + url, + { + method: 'DELETE', + headers: { ...this.headers } + } + ) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) + } + return await response.json() + } catch (error) { + return { data: null, error } } - return { data: null, error: null } } async listDatabases() { - const url = `${this.webliteUrl}/databases` - const response = await this.fetch(url, { method: 'GET' }) - if (!response.ok) { - return { data: null, error: new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) } + const url = `${this.baseUrl}/databases` + try { + const response = await this.fetch(url, { method: 'GET', headers: { ...this.headers } }) + if (!response.ok) { + throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) + } + return await response.json() + } catch (error) { + return { data: null, error } } - return { data: await response.json(), error: null } } - async create(dbName: string) { - const response = await fetch(`${this.webliteUrl}/sql?sql=CREATE DATABASE ${dbName}`, { method: 'POST' }) - if (!response.ok) { - return { data: null, error: new SQLiteCloudError(`Failed to create database: ${response.statusText}`) } + async createDatabase(filename: string) { + try { + const response = await this.sql`CREATE DATABASE ${filename}` + if (!response.ok) { + throw new SQLiteCloudError(`Failed to create database: ${response.statusText}`) + } + return await response.json() + } catch (error) { + return { data: null, error } } - return { data: null, error: null } } } From d90d3579f0885e7f1fac74ce67568d0412eea119 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Thu, 26 Dec 2024 01:00:53 -0800 Subject: [PATCH 15/18] add weblite tests --- src/packages/test/client.test.ts | 2 +- src/packages/test/weblite.test.ts | 45 +++++++++++++++++++++++++++ src/packages/weblite/WebliteClient.ts | 34 ++++++++++---------- 3 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 src/packages/test/weblite.test.ts diff --git a/src/packages/test/client.test.ts b/src/packages/test/client.test.ts index 817e58f..77f0c1e 100644 --- a/src/packages/test/client.test.ts +++ b/src/packages/test/client.test.ts @@ -19,7 +19,7 @@ describe('SQLiteCloudClient test suite', () => { it('should be able to query the database', async () => { const { data, error } = await client.sql`SELECT * FROM ${DEFAULT_TABLE_NAME}`; - + console.log(data, error) expect(data).toBeDefined() expect(error).toBeNull() }) diff --git a/src/packages/test/weblite.test.ts b/src/packages/test/weblite.test.ts new file mode 100644 index 0000000..da10645 --- /dev/null +++ b/src/packages/test/weblite.test.ts @@ -0,0 +1,45 @@ +import { CHINOOK_DATABASE_URL } from "../../../test/shared"; +import { WebliteClient } from "../weblite/WebliteClient"; + +const client = new WebliteClient(CHINOOK_DATABASE_URL) + +describe('WebliteClient test suite', () => { + const DATABASE_NAME = `${Date.now()}.sqlite` + + it('should be able to create a client', () => { + expect(client).toBeDefined() + expect(client).toBeInstanceOf(WebliteClient) + }) + + it('should be able to create a database', async () => { + const { data, error } = await client.createDatabase(DATABASE_NAME) + expect(data).toBeDefined() + expect(error).toBeNull() + }) + + it('should be able to list databases', async () => { + const { data, error } = await client.listDatabases() + expect(data).toBeDefined() + expect(data.length).toBeGreaterThan(0) + expect(error).toBeNull() + }) + + it('should be able to download and upload a database', async () => { + const { data, error } = await client.downloadDatabase(DATABASE_NAME) + expect(data).toBeDefined() + expect(error).toBeNull() + expect(data).toBeInstanceOf(ArrayBuffer) + const response = await client.uploadDatabase(DATABASE_NAME + '_upload', Buffer.from(data as ArrayBuffer)) + expect(response.data).toBeTruthy() + expect(response.error).toBeNull() + }) + + it('should be able to delete a database', async () => { + const deleteResponse = await client.deleteDatabase(DATABASE_NAME) + const deleteCopyResponse = await client.deleteDatabase(DATABASE_NAME + '_upload') + expect(deleteResponse.data).toBeDefined() + expect(deleteResponse.error).toBeNull() + expect(deleteCopyResponse.data).toBeDefined() + expect(deleteCopyResponse.error).toBeNull() + }) +}) diff --git a/src/packages/weblite/WebliteClient.ts b/src/packages/weblite/WebliteClient.ts index 933867f..0d91f36 100644 --- a/src/packages/weblite/WebliteClient.ts +++ b/src/packages/weblite/WebliteClient.ts @@ -31,6 +31,7 @@ export class WebliteClient { try { let _sql = '' if (Array.isArray(sql) && 'raw' in sql) { // check raw property? + _sql = this._defaultDatabase ? `USE DATABASE ${this._defaultDatabase}; ` : ''; sql.forEach((string, i) => { // TemplateStringsArray splits the string before each variable // used in the template. Add the question mark @@ -52,7 +53,7 @@ export class WebliteClient { if (!response.ok) { throw new SQLiteCloudError(`Failed to execute sql: ${response.statusText}`) } - return await response.json() + return { error: null, ...(await response.json()) } } catch (error) { return { data: null, error } } @@ -61,12 +62,17 @@ export class WebliteClient { get defaultDatabase() { return this._defaultDatabase } - + // Set default database for .sql() calls useDatabase(name: string) { this._defaultDatabase = name return this } + + async createDatabase(filename: string) { + return await this.sql`CREATE DATABASE ${filename}`; + } + async uploadDatabase( filename: string, database: File | Buffer | Blob | string, @@ -88,6 +94,7 @@ export class WebliteClient { } const headers = { + 'Content-Type': 'application/octet-stream', ...(opts.headers ?? {}), ...this.headers, ...DEFAULT_HEADERS, @@ -100,7 +107,8 @@ export class WebliteClient { if (!response.ok) { throw new SQLiteCloudError(`Failed to upload database: ${response.statusText}`) } - return await response.json() + + return { error: null, ...(await response.json()) } } catch (error) { return { data: null, error } } @@ -116,8 +124,10 @@ export class WebliteClient { if (!response.ok) { throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`) } + const isNode = typeof window === 'undefined' - return isNode ? await response.arrayBuffer() : await response.blob() + const data = isNode ? await response.arrayBuffer() : await response.blob() + return { error: null, data } } catch (error) { return { data: null, error } } @@ -137,7 +147,7 @@ export class WebliteClient { if (!response.ok) { throw new SQLiteCloudError(`Failed to delete database: ${response.statusText}`) } - return await response.json() + return { error: null, ...(await response.json()) } } catch (error) { return { data: null, error } } @@ -150,19 +160,7 @@ export class WebliteClient { if (!response.ok) { throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) } - return await response.json() - } catch (error) { - return { data: null, error } - } - } - - async createDatabase(filename: string) { - try { - const response = await this.sql`CREATE DATABASE ${filename}` - if (!response.ok) { - throw new SQLiteCloudError(`Failed to create database: ${response.statusText}`) - } - return await response.json() + return { error: null, ...(await response.json()) } } catch (error) { return { data: null, error } } From 6fbf85ca4a4c8ebcff9258557f1b99803264bdbf Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Thu, 26 Dec 2024 01:02:42 -0800 Subject: [PATCH 16/18] client test clean up --- src/packages/test/client.test.ts | 4 +--- src/packages/test/storage.test.ts | 37 ------------------------------- src/packages/test/utils.ts | 0 3 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 src/packages/test/storage.test.ts delete mode 100644 src/packages/test/utils.ts diff --git a/src/packages/test/client.test.ts b/src/packages/test/client.test.ts index 77f0c1e..29a489c 100644 --- a/src/packages/test/client.test.ts +++ b/src/packages/test/client.test.ts @@ -17,11 +17,9 @@ describe('SQLiteCloudClient test suite', () => { expect(() => new SQLiteCloudClient({ connectionString: 'invalid' })).toThrow() }) - it('should be able to query the database', async () => { + it('should be able to query the database via HTTP', async () => { const { data, error } = await client.sql`SELECT * FROM ${DEFAULT_TABLE_NAME}`; - console.log(data, error) expect(data).toBeDefined() expect(error).toBeNull() }) - }) diff --git a/src/packages/test/storage.test.ts b/src/packages/test/storage.test.ts deleted file mode 100644 index dbf38bc..0000000 --- a/src/packages/test/storage.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { expect } from '@jest/globals' -import { StorageClient } from '../_storage/StorageClient' -import { CHINOOK_DATABASE_URL } from '../../../test/shared' - -const TEST_BUCKET_NAME = 'test_bucket' - -const storage = new StorageClient(CHINOOK_DATABASE_URL, - { - headers: { - 'Content-Type': 'application/json' - } - } -) - -describe('StorageClient', () => { - it('should be able to create a bucket', async () => { - expect(storage).toBeDefined() - const getBucketResponse = await storage.getBucket(TEST_BUCKET_NAME) - console.log(getBucketResponse) - - const { data, error } = await storage.createBucket(TEST_BUCKET_NAME) - console.log(data, error) - expect(error).toBeNull() - expect(data).toBeDefined() - }) - - it('should get a bucket', async () => { - expect(storage).toBeDefined() - - const { data, error } = await storage.getBucket(TEST_BUCKET_NAME) - console.log(data) - expect(error).toBeNull() - expect(data).toBeDefined() - }) - - -}) diff --git a/src/packages/test/utils.ts b/src/packages/test/utils.ts deleted file mode 100644 index e69de29..0000000 From 32b98f16da7f19774c4203a73c88afec04617681 Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Thu, 26 Dec 2024 11:21:41 -0800 Subject: [PATCH 17/18] clean up weblite and start on pubsub refactor --- package-lock.json | 4 +- src/drivers/database.ts | 3 +- src/packages/SQLiteCloudClient.ts | 29 ++-- src/packages/_pubsub/PubSubClient.ts | 62 +++----- src/packages/_storage/StorageClient.ts | 2 +- src/packages/test/client.test.ts | 14 +- src/packages/test/pubsub.test.ts | 210 +++++++++++++++++++++++++ src/packages/weblite/WebliteClient.ts | 6 +- test/SQLiteCloudClient.test.ts | 0 test/database.test.ts | 2 - 10 files changed, 263 insertions(+), 69 deletions(-) create mode 100644 src/packages/test/pubsub.test.ts delete mode 100644 test/SQLiteCloudClient.test.ts diff --git a/package-lock.json b/package-lock.json index 97d6b4e..71ce59f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.354", + "version": "1.0.360", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@sqlitecloud/drivers", - "version": "1.0.354", + "version": "1.0.360", "license": "MIT", "dependencies": { "@craftzdog/react-native-buffer": "^6.0.5", diff --git a/src/drivers/database.ts b/src/drivers/database.ts index 16e5920..fb04c0d 100644 --- a/src/drivers/database.ts +++ b/src/drivers/database.ts @@ -472,7 +472,8 @@ export class Database extends EventEmitter { } else { // metadata for operations like insert, update, delete? const context = this.processContext(results) - resolve(context ? context : results) + const result = { data: context ? context : results, error: null } + resolve(result) } }) } diff --git a/src/packages/SQLiteCloudClient.ts b/src/packages/SQLiteCloudClient.ts index 6fa97ab..08d2427 100644 --- a/src/packages/SQLiteCloudClient.ts +++ b/src/packages/SQLiteCloudClient.ts @@ -2,10 +2,8 @@ import { Database } from '../drivers/database' import { Fetch, fetchWithAuth } from './utils/fetch' import { PubSubClient } from './_pubsub/PubSubClient' import { WebliteClient } from './weblite/WebliteClient' -import { StorageClient } from './_storage/StorageClient' import { SQLiteCloudDataTypes, SQLiteCloudError } from '../drivers/types' import { cleanConnectionString } from './utils' -import { FunctionsClient } from './_functions/FunctionsClient' import { SQLiteCloudClientConfig } from './types' import { DEFAULT_HEADERS } from '../drivers/constants' @@ -61,13 +59,6 @@ export class SQLiteCloudClient { return await this.weblite.sql(sql, ...values) } - get pubSub() { - if (!this._pubSub) { - this._pubSub = new PubSubClient(this.db.getConfiguration()) - } - return this._pubSub - } - get db() { if (!this._db) { this._db = new Database(this.connectionString) @@ -85,18 +76,18 @@ export class SQLiteCloudClient { return this._weblite } - get files() { - return new StorageClient(this.connectionString, { - customFetch: this.fetch, - headers: this._globalHeaders - }) + get pubSub() { + if (!this._pubSub) { + this._pubSub = new PubSubClient( + this.db + ) + } + return this._pubSub } - get functions() { - return new FunctionsClient(this.connectionString, { - customFetch: this.fetch, - headers: this._globalHeaders - }) + close() { + if (this._db) this._db.close() + if (this._pubSub) this._pubSub.close() } } diff --git a/src/packages/_pubsub/PubSubClient.ts b/src/packages/_pubsub/PubSubClient.ts index 1a275c6..6d6512d 100644 --- a/src/packages/_pubsub/PubSubClient.ts +++ b/src/packages/_pubsub/PubSubClient.ts @@ -1,6 +1,8 @@ import { SQLiteCloudConnection } from '../../drivers/connection' import SQLiteCloudTlsConnection from '../../drivers/connection-tls' +import { Database } from '../../drivers/database' import { SQLiteCloudConfig } from '../../drivers/types' +import { getDefaultDatabase } from '../utils' /** * PubSubCallback @@ -33,14 +35,13 @@ export interface ListenOptions { * @param close - Close the connection. */ export interface PubSub { - listen(options: ListenOptions, callback: PubSubCallback): Promise + listen(options: ListenOptions, callback: PubSubCallback): Promise unlisten(options: ListenOptions): void subscribe(channelName: string, callback: PubSubCallback): Promise unsubscribe(channelName: string): void create(channelName: string, failIfExists: boolean): Promise delete(channelName: string): Promise notify(channelName: string, message: string): Promise - setPubSubOnly(): Promise connected(): boolean close(): void } @@ -50,13 +51,15 @@ export interface PubSub { */ export class PubSubClient implements PubSub { protected _pubSubConnection: SQLiteCloudConnection | null - protected defaultDatabaseName: string + protected _queryConnection: Database + protected defaultDatabase: string protected config: SQLiteCloudConfig - constructor(config: SQLiteCloudConfig) { - this.config = config + constructor(conn: Database) { + this.config = conn.getConfiguration() + this.defaultDatabase = this.config.database ?? '' + this._queryConnection = conn this._pubSubConnection = null - this.defaultDatabaseName = config?.database ?? '' } /** @@ -66,16 +69,16 @@ export class PubSubClient implements PubSub { * @param callback Callback to be called when a message is received */ - get pubSubConnection(): SQLiteCloudConnection { + private get pubSubConnection(): SQLiteCloudConnection { if (!this._pubSubConnection) { this._pubSubConnection = new SQLiteCloudTlsConnection(this.config) } return this._pubSubConnection } - async listen(options: ListenOptions, callback: PubSubCallback): Promise { - const _dbName = options.dbName ? options.dbName : this.defaultDatabaseName; - const authCommand: string = await this.pubSubConnection.sql`LISTEN ${options.tableName} DATABASE ${_dbName};` + async listen(options: ListenOptions, callback: PubSubCallback): Promise { + const _dbName = options.dbName ? options.dbName : this.defaultDatabase; + const authCommand: string = await this._queryConnection.sql`LISTEN TABLE ${options.tableName} DATABASE ${_dbName};` return new Promise((resolve, reject) => { this.pubSubConnection.sendCommands(authCommand, (error, results) => { @@ -85,6 +88,7 @@ export class PubSubClient implements PubSub { } else { // skip results from pubSub auth command if (results !== 'OK') { + console.log(results) callback.call(this, null, results) } resolve(results) @@ -97,8 +101,8 @@ export class PubSubClient implements PubSub { * Unlisten to a table. * @param options Options for the unlisten operation. */ - public unlisten(options: ListenOptions): void { - this.pubSubConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` + public async unlisten(options: ListenOptions): Promise { + return this._queryConnection.sql`UNLISTEN ${options.tableName} DATABASE ${options.dbName};` } /** @@ -107,7 +111,7 @@ export class PubSubClient implements PubSub { * @param callback Callback to be called when a message is received. */ public async subscribe(channelName: string, callback: PubSubCallback): Promise { - const authCommand: string = await this.pubSubConnection.sql`LISTEN ${channelName};` + const authCommand: string = await this._queryConnection.sql`LISTEN ${channelName};` return new Promise((resolve, reject) => { this.pubSubConnection.sendCommands(authCommand, (error, results) => { @@ -125,8 +129,8 @@ export class PubSubClient implements PubSub { * Unsubscribe (unlisten) from a channel. * @param channelName The name of the channel to unsubscribe from. */ - public unsubscribe(channelName: string): void { - this.pubSubConnection.sql`UNLISTEN ${channelName};` + public async unsubscribe(channelName: string): Promise { + return this._queryConnection.sql`UNLISTEN ${channelName};` } /** @@ -135,7 +139,7 @@ export class PubSubClient implements PubSub { * @param failIfExists Raise an error if the channel already exists */ public async create(channelName: string, failIfExists: boolean = true): Promise { - return await this.pubSubConnection.sql( + return this._queryConnection.sql( `CREATE CHANNEL ?${failIfExists ? '' : ' IF NOT EXISTS'};`, channelName ) } @@ -145,34 +149,14 @@ export class PubSubClient implements PubSub { * @param name Channel name */ public async delete(channelName: string): Promise { - return await this.pubSubConnection.sql`REMOVE CHANNEL ${channelName};` + return this._queryConnection.sql`REMOVE CHANNEL ${channelName};` } /** * Send a message to the channel. */ - public notify(channelName: string, message: string): Promise { - return this.pubSubConnection.sql`NOTIFY ${channelName} ${message};` - } - - // DOUBLE CHECK THIS - - /** - * Ask the server to close the connection to the database and - * to keep only open the Pub/Sub connection. - * Only interaction with Pub/Sub commands will be allowed. - */ - public setPubSubOnly(): Promise { - return new Promise((resolve, reject) => { - this.pubSubConnection.sendCommands('PUBSUB ONLY;', (error, results) => { - if (error) { - reject(error) - } else { - this.pubSubConnection.close() - resolve(results) - } - }) - }) + public async notify(channelName: string, message: string) { + return await this._queryConnection.sql`NOTIFY ${channelName} ${message};` } /** True if Pub/Sub connection is open. */ diff --git a/src/packages/_storage/StorageClient.ts b/src/packages/_storage/StorageClient.ts index a1dfe09..b7c6846 100644 --- a/src/packages/_storage/StorageClient.ts +++ b/src/packages/_storage/StorageClient.ts @@ -23,7 +23,7 @@ export class StorageClient implements Storage { this.filesUrl = getAPIUrl(connectionString, 'files') this.webliteSQLUrl = getAPIUrl(connectionString, 'weblite/sql') this.fetch = options.customFetch || fetchWithAuth(connectionString) - this.headers = { ...DEFAULT_HEADERS, ...options.headers } + this.headers = { ...options.headers } } async createBucket(bucket: string) { diff --git a/src/packages/test/client.test.ts b/src/packages/test/client.test.ts index 29a489c..d6c0298 100644 --- a/src/packages/test/client.test.ts +++ b/src/packages/test/client.test.ts @@ -3,10 +3,11 @@ import { SQLiteCloudClient } from '../SQLiteCloudClient' const DEFAULT_TABLE_NAME = 'albums'; -const client = new SQLiteCloudClient(CHINOOK_DATABASE_URL) + describe('SQLiteCloudClient test suite', () => { it('should be able to create a client', () => { + const client = new SQLiteCloudClient(CHINOOK_DATABASE_URL) expect(client).toBeDefined() expect(client).toBeInstanceOf(SQLiteCloudClient) }) @@ -18,8 +19,17 @@ describe('SQLiteCloudClient test suite', () => { }) it('should be able to query the database via HTTP', async () => { + const client = new SQLiteCloudClient(CHINOOK_DATABASE_URL) const { data, error } = await client.sql`SELECT * FROM ${DEFAULT_TABLE_NAME}`; expect(data).toBeDefined() expect(error).toBeNull() }) -}) + + it('should be able to query via database connection', async () => { + const client = new SQLiteCloudClient(CHINOOK_DATABASE_URL) + const { data, error } = await client.db.sql('SELECT * FROM albums') + expect(data).toBeDefined() + expect(error).toBeNull() + client.close() + }) +}) \ No newline at end of file diff --git a/src/packages/test/pubsub.test.ts b/src/packages/test/pubsub.test.ts new file mode 100644 index 0000000..878433a --- /dev/null +++ b/src/packages/test/pubsub.test.ts @@ -0,0 +1,210 @@ + +import { PubSubClient } from '../_pubsub/PubSubClient' +import { CHINOOK_DATABASE_URL, getChinookDatabase, LONG_TIMEOUT } from '../../../test/shared' +import { Database } from '../../drivers/database' + +const TABLE_NAME = 'albums'; + +describe('pubSub', () => { + it( + 'should listen, notify and receive pubSub messages on channel', + async () => { + const connection = getChinookDatabase() + const pubSub = new PubSubClient(connection) + + try { + const channelName = 'test-channel-' + crypto.randomUUID() + let callbackCalled = false + const message = 'Message in a bottle ' + Math.floor(Math.random() * 999) + + await pubSub.create(channelName, false) + + await pubSub.subscribe( + channelName, + (error, results) => { + expect(error).toBeNull() + expect(results).not.toBeNull() + expect(results['channel']).toEqual(channelName) + expect(results['payload']).toEqual(message) + callbackCalled = true + }, + ) + + await pubSub.notify(channelName, message) + + while (!callbackCalled) { + await new Promise(resolve => setTimeout(resolve, 1000)) + } + + expect(callbackCalled).toBeTruthy() + pubSub.delete(channelName) + } finally { + connection.close() + pubSub.close() + } + }, + ) + // it('should unlisten on channel', async () => { + // const connection = getChinookDatabase() + // const pubSub = await connection.getPubSub() + + // try { + // const channelName = 'test-channel-' + Math.floor(Math.random() * 999) + + // await pubSub.createChannel(channelName, false) + + // await pubSub.listen(PUBSUB_ENTITY_TYPE.CHANNEL, channelName, (error, results, data) => { + // expect(true).toBeFalsy() + // }) + + // let connections = await connection.sql`LIST PUBSUB CONNECTIONS;` + // let connectionExists = connections.find((row: SQLiteCloudRow) => row['chname'] === channelName) + // expect(connectionExists).toBeDefined() + + // await pubSub.unlisten(PUBSUB_ENTITY_TYPE.CHANNEL, channelName) + + // connections = await connection.sql`LIST PUBSUB CONNECTIONS;` + // connectionExists = connections.find((row: SQLiteCloudRow) => row['chname'] === channelName) + // expect(connectionExists).toBeUndefined() + // } finally { + // connection.close() + // pubSub.close() + // } + // }), + // it('should unlisten on table', async () => { + // const connection = getChinookDatabase() + // const pubSub = await connection.getPubSub() + + // try { + // let callbackCalled = false + + // const tableName = 'genres' + // await pubSub.listen(PUBSUB_ENTITY_TYPE.TABLE, tableName, (error, results, data) => { + // expect(true).toBeFalsy() + // callbackCalled = true + // }) + + // let connections = await connection.sql`LIST PUBSUB CONNECTIONS;` + // let connectionExists = connections.find((row: SQLiteCloudRow) => row['chname'] === tableName) + // expect(connectionExists).toBeDefined() + + // await pubSub.unlisten(PUBSUB_ENTITY_TYPE.TABLE, tableName) + + // await connection.sql`UPDATE genres SET Name = 'Rock' WHERE GenreId = 1` + + // // wait a moment to see if the callback is called + // await new Promise(resolve => setTimeout(resolve, 2000)) + + // expect(callbackCalled).toBeFalsy() + // } finally { + // connection.close() + // pubSub.close() + // } + // }), + // it('should fail to create a channel that already exists', async () => { + // const connection = getChinookDatabase() + // const pubSub = await connection.getPubSub() + + // try { + // const channelName = 'test-channel-' + crypto.randomUUID() + + // await pubSub.createChannel(channelName) + + // await expect(pubSub.createChannel(channelName, true)).rejects.toThrow(`Cannot create channel ${channelName} because it already exists.`) + // } finally { + // connection.close() + // pubSub.close() + // } + // }), + // it( + // 'should listen and receive pubSub messages on table', + // async () => { + // const connection = getChinookDatabase() + // const pubSub = await connection.getPubSub() + + // try { + // let callbackCalled = false + // const newName = 'Rock' + Math.floor(Math.random() * 999) + + // await pubSub.listen( + // PUBSUB_ENTITY_TYPE.TABLE, + // 'genres', + // (error, results, data) => { + // expect(error).toBeNull() + + // expect(results).not.toBeNull() + // expect(results['payload'][0]['type']).toEqual('UPDATE') + // expect(results['payload'][0]['Name']).toEqual(newName) + // expect(data).toEqual({ pippo: 'pluto' }) + // callbackCalled = true + // }, + // { pippo: 'pluto' } + // ) + + // await connection.sql`UPDATE genres SET Name = ${newName} WHERE GenreId = 1` + + // while (!callbackCalled) { + // await new Promise(resolve => setTimeout(resolve, 1000)) + // } + + // expect(callbackCalled).toBeTruthy() + // } finally { + // connection.close() + // pubSub.close() + // } + // }, + // LONG_TIMEOUT + // ), + // it('should be connected', async () => { + // const connection = getChinookDatabase() + // const pubSub = await connection.getPubSub() + + // try { + // expect(pubSub.connected()).toBeTruthy() + + // pubSub.close() + + // expect(pubSub.connected()).toBeFalsy() + // } finally { + // connection.close() + // pubSub.close() + // } + // }), + // it( + // 'should keep pubSub only connection', + // async () => { + // const connection = getChinookDatabase() + // const connection2 = getChinookDatabase() + // const pubSub = await connection.getPubSub() + + // try { + // let callbackCalled = false + // const newName = 'Rock' + Math.floor(Math.random() * 999) + + // await pubSub.listen(PUBSUB_ENTITY_TYPE.TABLE, 'genres', (error, results, data) => { + // expect(error).toBeNull() + // expect(results).not.toBeNull() + // callbackCalled = true + // }) + + // await pubSub.setPubSubOnly() + + // expect(connection.sql`SELECT 1`).rejects.toThrow('Connection not established') + // expect(pubSub.connected()).toBeTruthy() + + // await connection2.sql`UPDATE genres SET Name = ${newName} WHERE GenreId = 1` + + // while (!callbackCalled) { + // await new Promise(resolve => setTimeout(resolve, 1000)) + // } + + // expect(callbackCalled).toBeTruthy() + // } finally { + // connection.close() + // pubSub.close() + // connection2.close() + // } + // }, + // LONG_TIMEOUT + // ) +}) \ No newline at end of file diff --git a/src/packages/weblite/WebliteClient.ts b/src/packages/weblite/WebliteClient.ts index 0d91f36..b9ebc15 100644 --- a/src/packages/weblite/WebliteClient.ts +++ b/src/packages/weblite/WebliteClient.ts @@ -29,9 +29,9 @@ export class WebliteClient { const url = `${this.baseUrl}/sql` try { - let _sql = '' + let _sql = this._defaultDatabase ? `USE DATABASE ${this._defaultDatabase}; ` : ''; + if (Array.isArray(sql) && 'raw' in sql) { // check raw property? - _sql = this._defaultDatabase ? `USE DATABASE ${this._defaultDatabase}; ` : ''; sql.forEach((string, i) => { // TemplateStringsArray splits the string before each variable // used in the template. Add the question mark @@ -39,7 +39,7 @@ export class WebliteClient { _sql += string + (i < values.length ? '?' : '') }) } else if (typeof sql === 'string') { - _sql = sql + _sql = _sql + sql } else { throw new SQLiteCloudError('Invalid sql') } diff --git a/test/SQLiteCloudClient.test.ts b/test/SQLiteCloudClient.test.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test/database.test.ts b/test/database.test.ts index a22bc90..fc719ef 100644 --- a/test/database.test.ts +++ b/test/database.test.ts @@ -10,11 +10,9 @@ import { removeDatabase, removeDatabaseAsync, LONG_TIMEOUT, - getChinookWebsocketConnection } from './shared' import { RowCountCallback } from '../src/drivers/types' import { expect, describe, it } from '@jest/globals' -import { Database } from 'sqlite3' // // utility methods to setup and destroy temporary test databases From c6c85b08d2e06c024acba3a63f65452b6d53659a Mon Sep 17 00:00:00 2001 From: Jacob Prall Date: Thu, 26 Dec 2024 21:47:22 -0800 Subject: [PATCH 18/18] fix: add functions test --- package.json | 2 +- src/packages/_functions/FunctionsClient.ts | 49 +++++++++------------- src/packages/test/functions.test.ts | 44 +++++++++++++++++++ src/packages/utils/fetch.ts | 6 +-- src/packages/weblite/WebliteClient.ts | 8 ++-- 5 files changed, 71 insertions(+), 38 deletions(-) create mode 100644 src/packages/test/functions.test.ts diff --git a/package.json b/package.json index 33bb6ff..fa0e813 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.360", + "version": "1.0.361", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/packages/_functions/FunctionsClient.ts b/src/packages/_functions/FunctionsClient.ts index 0b0a738..e341953 100644 --- a/src/packages/_functions/FunctionsClient.ts +++ b/src/packages/_functions/FunctionsClient.ts @@ -10,8 +10,9 @@ import { Fetch, resolveFetch } from '../utils/fetch' * @param headers - The headers to pass to the function. */ interface FunctionInvokeOptions { - args: any[] + params: Record headers?: Record + apiKey?: string } /** @@ -27,13 +28,15 @@ export class FunctionsClient { constructor( connectionString: string, options: { - customFetch?: Fetch, + fetch?: Fetch, headers?: Record - } = {} + } = { + headers: {} + } ) { this.url = getAPIUrl(connectionString, FUNCTIONS_ROOT_PATH) - this.fetch = resolveFetch(options.customFetch) - this.headers = options.headers ? { ...DEFAULT_HEADERS, ...options.headers } : { ...DEFAULT_HEADERS } + this.fetch = resolveFetch(options.fetch) + this.headers = { ...DEFAULT_HEADERS, ...options.headers } } // TODO: check authorization and api key setup in Gateway setAuth(token: string) { @@ -43,56 +46,42 @@ export class FunctionsClient { async invoke(functionId: string, options: FunctionInvokeOptions) { let body; let _headers: Record = {} - if (options.args && + if (options.params && ((options.headers && !Object.prototype.hasOwnProperty.call(options.headers, 'Content-Type')) || !options.headers) ) { if ( - (typeof Blob !== 'undefined' && options.args instanceof Blob) || - options.args instanceof ArrayBuffer + (typeof Blob !== 'undefined' && options.params instanceof Blob) || + options.params instanceof ArrayBuffer ) { // will work for File as File inherits Blob // also works for ArrayBuffer as it is the same underlying structure as a Blob _headers['Content-Type'] = 'application/octet-stream' - body = options.args - } else if (typeof options.args === 'string') { + body = options.params + } else if (typeof options.params === 'string') { // plain string _headers['Content-Type'] = 'text/plain' - body = options.args - } else if (typeof FormData !== 'undefined' && options.args instanceof FormData) { + body = options.params + } else if (typeof FormData !== 'undefined' && options.params instanceof FormData) { _headers['Content-Type'] = 'multipart/form-data' - body = options.args + body = options.params } else { // default, assume this is JSON _headers['Content-Type'] = 'application/json' - body = JSON.stringify(options.args) + body = JSON.stringify(options.params) } } try { const response = await this.fetch(`${this.url}/${functionId}`, { method: 'POST', - body: JSON.stringify(options.args), + body, headers: { ..._headers, ...this.headers, ...options.headers } }) if (!response.ok) { throw new SQLiteCloudError(`Failed to invoke function: ${response.statusText}`) } - - let responseType = (response.headers.get('Content-Type') ?? 'text/plain').split(';')[0].trim() - let data: any - if (responseType === 'application/json') { - data = await response.json() - } else if (responseType === 'application/octet-stream') { - data = await response.blob() - } else if (responseType === 'text/event-stream') { - data = response - } else if (responseType === 'multipart/form-data') { - data = await response.formData() - } else { - data = await response.text() - } - return { ...data, error: null } + return { error: null, ...(await response.json()) } } catch (error) { return { data: null, error } } diff --git a/src/packages/test/functions.test.ts b/src/packages/test/functions.test.ts new file mode 100644 index 0000000..4c400da --- /dev/null +++ b/src/packages/test/functions.test.ts @@ -0,0 +1,44 @@ + +// Test functions client +// invoke + +import { CHINOOK_API_KEY, CHINOOK_DATABASE_URL } from "../../../test/shared" +import { FunctionsClient } from "../_functions/FunctionsClient" + +const TEST_SQL_FUNCTION_ID = 'test-1-sql' +const TEST_JS_FUNCTION_ID = 'test-1-js' + +const TEST_FUNCTION_ARG = { + filter: 'a', + limit: 10 +} + +const functions = new FunctionsClient(CHINOOK_DATABASE_URL) + +describe('FunctionsClient', () => { + it('should invoke a JS function', async () => { + + const { data, error } = await functions.invoke(TEST_JS_FUNCTION_ID, { + params: TEST_FUNCTION_ARG, + headers: { + 'Authorization': `Bearer ${CHINOOK_API_KEY}` + } + }) + expect(data.message).toBeDefined() + expect(data.result).toBeDefined() + expect(error).toBeNull() + }) + + it('should invoke a SQL function', async () => { + const { data, error } = await functions.invoke(TEST_SQL_FUNCTION_ID, { + params: TEST_FUNCTION_ARG, + headers: { + 'Authorization': `Bearer ${CHINOOK_API_KEY}` + } + }) + expect(data).toBeDefined() + expect(data.length > 0).toBeTruthy() + expect(error).toBeNull() + }) +}) + diff --git a/src/packages/utils/fetch.ts b/src/packages/utils/fetch.ts index 4a11340..9901f28 100644 --- a/src/packages/utils/fetch.ts +++ b/src/packages/utils/fetch.ts @@ -7,11 +7,11 @@ export const resolveFetch = (customFetch?: Fetch): Fetch => { if (customFetch) { _fetch = customFetch } else if (typeof fetch !== 'undefined') { - _fetch = nodeFetch as unknown as Fetch - } else { _fetch = fetch + } else { + _fetch = nodeFetch as unknown as Fetch } - return (...args: Parameters) => _fetch(...args) + return _fetch } export const resolveHeadersConstructor = () => { diff --git a/src/packages/weblite/WebliteClient.ts b/src/packages/weblite/WebliteClient.ts index b9ebc15..20b30dc 100644 --- a/src/packages/weblite/WebliteClient.ts +++ b/src/packages/weblite/WebliteClient.ts @@ -21,7 +21,7 @@ export class WebliteClient { ) { this.baseUrl = getAPIUrl(connectionString, 'weblite') this.fetch = options?.fetch || fetchWithAuth(connectionString) - this.headers = { ...options.headers } + this.headers = { ...DEFAULT_HEADERS, ...options.headers } this._defaultDatabase = getDefaultDatabase(connectionString) } @@ -120,7 +120,7 @@ export class WebliteClient { const filenamePath = encodeURIComponent(filename) const url = `${this.baseUrl}/${filenamePath}` try { - const response = await this.fetch(url, { method: 'GET', headers: { ...this.headers } }) + const response = await this.fetch(url, { method: 'GET', headers: this.headers }) if (!response.ok) { throw new SQLiteCloudError(`Failed to download database: ${response.statusText}`) } @@ -141,7 +141,7 @@ export class WebliteClient { url, { method: 'DELETE', - headers: { ...this.headers } + headers: this.headers } ) if (!response.ok) { @@ -156,7 +156,7 @@ export class WebliteClient { async listDatabases() { const url = `${this.baseUrl}/databases` try { - const response = await this.fetch(url, { method: 'GET', headers: { ...this.headers } }) + const response = await this.fetch(url, { method: 'GET', headers: this.headers }) if (!response.ok) { throw new SQLiteCloudError(`Failed to list databases: ${response.statusText}`) }