From 57c85deb23be6270315985cbc1501c5b2857c4f5 Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Thu, 9 May 2024 16:12:16 -0400 Subject: [PATCH 01/13] pyc files cleanup --- .../agents/__pycache__/__init__.cpython-311.pyc | Bin 1290 -> 0 bytes .../agents/__pycache__/__init__.cpython-312.pyc | Bin 1137 -> 0 bytes .../agents/__pycache__/base.cpython-312.pyc | Bin 2040 -> 0 bytes .../dynamic_prompting.cpython-312.pyc | Bin 35851 -> 0 bytes .../__pycache__/generic_agent.cpython-312.pyc | Bin 7576 -> 0 bytes .../__pycache__/prompt_utils.cpython-312.pyc | Bin 1370 -> 0 bytes .../utils/__pycache__/__init__.cpython-312.pyc | Bin 163 -> 0 bytes .../utils/__pycache__/chat_api.cpython-311.pyc | Bin 13508 -> 0 bytes .../utils/__pycache__/chat_api.cpython-312.pyc | Bin 12529 -> 0 bytes .../utils/__pycache__/exp_utils.cpython-312.pyc | Bin 26737 -> 0 bytes .../utils/__pycache__/llm_utils.cpython-312.pyc | Bin 18449 -> 0 bytes .../prompt_templates.cpython-311.pyc | Bin 4235 -> 0 bytes .../prompt_templates.cpython-312.pyc | Bin 3692 -> 0 bytes 13 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 demo_agent/agents/__pycache__/__init__.cpython-311.pyc delete mode 100644 demo_agent/agents/__pycache__/__init__.cpython-312.pyc delete mode 100644 demo_agent/agents/__pycache__/base.cpython-312.pyc delete mode 100644 demo_agent/agents/__pycache__/dynamic_prompting.cpython-312.pyc delete mode 100644 demo_agent/agents/__pycache__/generic_agent.cpython-312.pyc delete mode 100644 demo_agent/agents/__pycache__/prompt_utils.cpython-312.pyc delete mode 100644 demo_agent/utils/__pycache__/__init__.cpython-312.pyc delete mode 100644 demo_agent/utils/__pycache__/chat_api.cpython-311.pyc delete mode 100644 demo_agent/utils/__pycache__/chat_api.cpython-312.pyc delete mode 100644 demo_agent/utils/__pycache__/exp_utils.cpython-312.pyc delete mode 100644 demo_agent/utils/__pycache__/llm_utils.cpython-312.pyc delete mode 100644 demo_agent/utils/__pycache__/prompt_templates.cpython-311.pyc delete mode 100644 demo_agent/utils/__pycache__/prompt_templates.cpython-312.pyc diff --git a/demo_agent/agents/__pycache__/__init__.cpython-311.pyc b/demo_agent/agents/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index beea2f31123dc874603a4008584d566c6b6da06c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1290 zcmZuw&1=*^6rV}5-~FrKWA}FdawTqYr3s(_`Ej5i`EEw-*HUud+J{!V2QZ z#0;6r9mG|xF+2=brkE+722GWkf_R2!H?&m@DBw?qnd4dTNX7o?C1mEA0;n=kZe)hn z0vR^SQz?=fb@@cpYXKzUN=$^pDxmCTe45ssfJ{>>@Weuto}HIV)?O5dB`YW~?>+^# zf)>$3n3KWqY+NI8f+KH%5*Hp-!((6KA)I3K?Fyw=PQgqu3Q%0JS>hcxS+aT5}=Yqtq@|@?0^tsZXwFMbe1Q}jnYV`b;iC`Lh%W( zCTjKczw%l~PQ4ji9&Ghp3=h)w(KE}7;n69e&tqE`uEG3e?CbT!6825Cj`V*YD2J!3 zYr)1dm=f9?nA{qe4ErW`Aq3J3QI^CPhL9*vJ|S@fguE_7b~@6SZ3S^9MNt}iBr8eQFIlF;)t6(TW&mw!MqA6IZs9gMoF~C$+zCA$751Jj z_l3Nu3F@ADU^~7MKNdP?C;7x!590rw9ZrgKNt}eu^2PY`z}hOtxP{JzhtdD`b{Z*# aw$gI diff --git a/demo_agent/agents/__pycache__/__init__.cpython-312.pyc b/demo_agent/agents/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 025ae40b53d340f22289b3847213405e30ad4803..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1137 zcmZuw&1(}u6rb7MY`&T%Z9?lsjQG{PBnb6nt<+XqDE0$U%7P5T&ZMs0&DPnCC85%y z74)KBqz4r|cxb7hNB;!HgDDlPt3}0=x0W1)o}AfDEJf$=oA-Nf-n`$wdHX(YJzz6UWhyyR#ijm0wYd!%)eI~cM zaT{u}F~4n?mMJrhpa)Dmmj+piZ!FHCmfUMdzLw6H9WT$Q;Y^p!8YQbhQ<+fY=YnHy zfp{8|yZ9WjD?Gy}b7$!bh zB=lzS9CPM)aJXu7YCB;vfe%QdgxQ{$a?Qcy?g5I(1+1?quf`|Ol@{_e(lOXKfU6;T3B7E z7tX#}s2?8R=$fc!CK~)Mpe6kn3g!kOKY_&5R4mHvHiT|euq7snGuic+pGF9mc_*=F zI%OB3pwmoD4O2My3NZ~YXc@~2XS;x$fKCC?7KjY@IETC7dnS8leDj#LIR5DBhN6AW z9$3kJ%oY|e+#A|Z3JoP1nS|jLB700z{p8<&kGf1uYgoIz;+d9P2qquTnxU$o;(iso vvA*yV``JOBEXLnG!C_m55PkzaTcBrK1yb_2)CMQvHW1j5xnn1A1X}(8^|chV diff --git a/demo_agent/agents/__pycache__/base.cpython-312.pyc b/demo_agent/agents/__pycache__/base.cpython-312.pyc deleted file mode 100644 index 8cb9b99aa7d6196a3925c9f18434fc951a768f9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2040 zcmah~&2Jk;6rcUF9mfu#xDu&S7}27#CD9O z7Kb9`Q2v5&>@ig?`9C;?+*UyxKth~wak=!wd$Vh&2}sOg=Djy>XWsk0-@N^?*{l;7 zn}^T%kx9s}s7!}d3X@wP><~tn7LZWuX`$ZJHPrRM2+f`uT0KkYMoy{blZa+@%7 zn=p%Odq#0iuQsgM)jv?N_1sv!xt1@u{ZtCqlOdNIF$442LEuWsBgfs6f^$d44tV5e zre+q0rOnpzdLpujI~y_#yyA{xu9nJ#fv`h3>1mAgbf)zTZn8O6htRCO(y3>G&Egei z@G7tI%0OR&oq@?J+vdz?uJovYy}E7o>fk*;&{+-S#!$ECb8B^-M{<a z{4nKWeHc<6J@7>wh2Ys%_jU7%;BZLf>19^a1+kw%VF8ArX6vJi${%CA%Tq3kVAP%M z`fRKF6f0z=JQ(22S&GL}`r_lxJqRJ~gpiXT#%<5_Jm0<2Vf;boW)a`*VF+p%JL=Gm zn$u3-O?f*R=9a))NAs(gWm_o?I6B3OJtE@vz-BHnRC-${Zj#ZvguAg0XsVK6Ozv&7D>!~i9u!u7-m4y+n zAZGP-E}bHY(PDw{v86Tj{*H)OK|TR45=1d&6NDWiHCWX6R%04IhrQAQx_%vWiQZ~} z#W>|*DvxRRAo$HG4+}l88;tBv_`uBqNfW^n5qlhn1)MKYc&TRXypC@_?r(BW=2g`P zwq)Kc`$TE4Pum7;&8zOfRD0;!hN!@u0;3YhHZcQ)(gMDP;}GDQbkr&!JZXIsK20(D zN94DqcX!wJ8wV>N9WLE?Y&^d6{ld}mmHi9*@xlAI4wr9(>GOly($V=#yT{elWph+%T=ylTV+6 zT%@ks>;r2a&)m16wt>aGK&ZHltZD3nLOr&W3nyO#ghUX9_(orr2WmPsOGkj6I qUf_5`T!xRacIi2o#)hV8zmN;Rlh1x8m!DVO(prz#{wA;}bp8SM^%EBW diff --git a/demo_agent/agents/__pycache__/dynamic_prompting.cpython-312.pyc b/demo_agent/agents/__pycache__/dynamic_prompting.cpython-312.pyc deleted file mode 100644 index a88eea901f19e930027ed7bf2c47c83dcd392adf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 35851 zcmd7532+-%nkJZu`y#=c;3XzSUDQEY5@lPrC{dR!(T6D8g35*=W|D*if~*86i73jZ zt5sX1%hi@=w!`#Rc};h@qp7uLTi)v3_D)U2$kEZ;9n(G0fWT^^QSyvt#uGE1lM?B6 zmuEJ5zyH0=L;?ccc2(5Gro@*oU%q_j|Ni%1@5i5)m3ahQ8#DKmzY7b(zo7^H@@tW$ zS8amurXUNl7!`v2E(S&RYzbP}-5RvwZi(84?LoUpd8|>#uruf!b_HGR*%oyVdx9Po zw@1ChzMzlA9nrGk@?bfOJEQ*LieLqcyP}oDRlzD2cSoy-Yl1bywZYoqx?mkk^F-^1 z1Hk}`d-1*@*dPju%OV^TWZyMGE_>g?+yB1SjbNiiP#WcMM3Bp6|34Jv3b_(t6^mE1 zcnyo!vUnW}>sc6JVFL>r{Lz zS|c z;6v=)3A}sERNFR|`Z!Wgno_s3)Kf_9HKp!Qy5uJ=ZV&|Zx8x^-JLRX8ofn4r44Fgu z>GwtcET5M9G%$JmrJX`(+wj?Lp4H_}ipBqCBeXQg+LNvgd;r-*@o$7n0_F@wna&Q5YA;#g6{Z zD7p8Bm~+XBGTJ{fI^JQ)Il^%{(x1q=QlYjwgmjV)-J)oaaf( zdqh=7)P8f9we=?~y@VNkQ&0rLv_Z=mVBersu_<=hLW^0p1|96l#-7BWlReqBcP{qi z(4O4v$*Dbg*po|p^0Ftl_T*zv9_^`&J$bdKa`xn7Ph~;BQh~OVvnMfF$=><3oK@_p zLVK!aPnGPcDp;e`%GC|%0lu!$a@4ULwd_d@*0ZNN?OlL9)w8ESut90WcMaORCibq8 zJv9ZJQF60Za*LLuh2>bqo?3&ekVDdPw6V0+Sf#BpaA#0b+T=CY1Y@^YJs#*-o3kH{ zVjlZ)t}$(1kLJ7-4POZ*R7J^E$?C|c{uCOBsPRP3M;Q`Bk=VsZY%o_&PYO+QC^|A2 z&sEd&XgHoQC$ch!BJsqCI-aXy6)>0#>1iHH8&-ztJ!^{`85qcUN7b>IA~S%?xjER4 zk1Dwu%H2N_>sOToD!&pQ9*t_LVGZVu)Ux5RXd)8AGQb*$=d9-=a)&MF7-ayfuEs>= z?ENDns{ExJuQsu0&d0Nd;-leyCFhRBLs&_)`dDw{s2A&GXe5#IDzWhSC>k>qPW0v~ zhcRxW>d5e@v6iZpD{KvhE=A%I%87M@Q4dG4ZmjBNs@~C$C9J4u+Q^k$xt^ajB%W*0 zpY`vRs4|TF@z6*tI-awR3=E*0ZB&j^RtCc8Y_67G85Qau0CQk`L&GDo(yt*?d?g69 zAn9)@fZ$CbVZ56Vbi{O8xFPncl<5Z0OI(Q##3`5b<;ksQ;))vIIt+9a86N5E5BK*+ z!rQjW%B8J`)RD`0d2D$ zmBf7rCWSlx`Wf2~UP`%Imhdcm^B#1FUv48W>Tf$4)=dE1`Lja5R#{yhy6@ z?Xif;#fxbpHAGUmI?8+c^<@sM#>0+o+lniIH_Q|WHI69F+rFR^*Q?y$kn&4 z`qOvC1l7+j*&0E<_dv#-7CLO|J|reLo{6c-VE@QqERs}YDS=hShG~G%Lx?sWX=H#W ze<^m}5IXER4*)tAPlRLriduy-ayGU;Ze4)^{$HGzmMy;ks+s==SM05U7C>)E8B*F}bghFZ!-l??|)KO4R0WBYukgwWE zK@$bd6s$s!vjTyrt@J2S&_=;J3S1PdrvOxtpmtEO5kbxxi6zuc^thFRhbZWxU^fK> zia9T%1BMmsy|Q|gk`Gcq;Ku=$=7miTwJzeRoB4ca1+tkWi&}yo-r7dNvy^ez0%*}fA0CFBnHFu?Jc4lk`vKzOiY;ChB zervYp1d6+ysy~?acV}!T?%PEBs{0O$y?N0kI4bX1oc8u5s~+Qn$Ht8SppQ-U(Qh=@ zY732eskx@zhEMZa#QO1PTr{6_ElncGlSM=n$(EoEe;|y44!KOWDo)uZ+d;HDz_z#) zx9l`+>mEG2OwV3CyG_qNJbO&fWwIBBQ4B{AS2uGt5-|HlOi|Lg3S4M)=AC5`q6=^6UgHG-AcqClKdBUn9VM7Pj8IjJ9 zMWTodDO%B;5;ho#VGeqYJwKt2M5W6^k^Ui8SWyckDooodJEdn?!CITc32fo##}Z0h z3gbVObxDauuqjEG!{e+l)DR(3f;LzQs4Wrc*V+-^jHV@|0Y!;Q1K9nkF(a2i_C#dF zhm>LIawIV%=?IaPW~vYy&Ox*kn@?g~!u~s=hCvE-=CEmNjmQl#*PU&zsR!|%=V^RQB zL{RZg@FAheqGohTeIOSl{0FoQj1d$lF}#b9D*cgxNPj_T1Tqq+ve+QzTOLzEVo=s0 zL@r8}jn0Q;bXDC@>Gf+E*Hiqvi2+=SHs> z9E(0d22_G}I*VZjWn#+pz+8DvFOv<06J~lfg_4$tK*6k47_+v}xGCMZDdXRqa&7+2 z&D($e4b2;2amEGy51%%@;;)`IQ?3W6&GfQ(+Dxzh0cK1ktp63Wa|o{)9>Kl%x@sc* z^X!4@v=FnPWxgsB%K?nC`FJ=cGxUqCp7wj-*!+%4xHpfZfkxCbC?Mz3v9|gQ9t#jA zm;@cECIf|#nSb+R`l<~9LnqbsZ*F;G%arquxBB|Ad2jPy*R}k^$kef{ziz6W4}lpi z5;|n4R6=B>A>bSp?U%{jPqS@SxQldgK&m~cpR_Y(E8y`K+d4074#;8Z<0+# zy)lW7zNKF$hk%}+DSb}X)jQ4o&xF-8*r6YfJ%)mW`UlhGzu=_FHZdT zHQ}DcZr`|AF8C`ak7cWCCr`5Ap3P&IW^IcCZZ~a9bXyDvj`DjJi+uxe2<3|uBd!5y zPWu)u4Y9?l71QkYdMzJf_si|}y2Wyvy>6~&S-_2r3Ts|JT4r4Oq%LhoQl1-7tQUw~ zEV^WH2-j@-QV!Yy=eC^ZG$VD0mg+4Kme9uxvTTUF35hd^$Wg4K6M4dyeM#FVvHV0( zVEC~R^gY(J$DJJ+lVS=d-BqQ3OpQk%SsrI(oJ0%)C%70zr!xthv?DqlU_@dCW4ANa z)hSWY#63V#W~R!+tVohP5`nHJl)-Vf6`Q{`3glxzdC>`t6(L-O1~aK3NPQ_083u(2 zavq%=R29gvSPwKx5wxvQnl=g$lt-cuTKMDEiQW*!EfLm#W=W9=F;FwB8=~w09Lk6 z2?L@mUUP$u1rrPQ(l#Yr5rcN@Hujtw7&mq?3~r>Kpi#dFJQ;}6LjN5R@-qF6Fm0O< zeh6N9Lj1sxbcw=*1$>dQD@<7NMBlSZrgui^>sdN zJ5AMi^O1#(3z^pWBs}qMT$eL+u7ma z8|GDf%%G6%xH(UJG#W{;57kptt~IK}R0gUq(WC9cNF)X&1C!#~S-xC(Ou5Y98)_IJ z#5i>XAj~0{uuQTd55asD{V7u8{|JFWD9j`lS|3WcK6JBZq3fx1*HgD!pSoKUc=O~N zCl_iqrE4~2YPL)rxzi%OS3P^?CyjI8zS*^~qbH63TYJ)NM=~u(r+Tx#iUr@Q+rCxr zwa+R)**N!7diB1vZ{N>-kY2XFv+J#0vuEZG%=M<$?#VRmo$7h*BtCfk`tAVYY4$I&Uc4ZEHM)o9X z9Ki!9m*EVRIu9a?ukL;&8V!YjVO~ZmL-Pmu5M8EHDEO!J0>B{r?KROCc>aG8gsVWOrM|gpPZ&sY z!k#d|TUS4*)(QJnN9=S#E@KfHxi-t<1!HOF+cjbD5ngxpTL!W8p2u>%>YQ-O7UE20 zE7=0X#n*gDu~Nz#me;Cm$amGLw;c7>=NGhWOXzky!h~JV^CRJHqo+`P5Yf%93CD!> z1H0b7tF8&xgm@8Ko@$%0_gh5J${3Fl2tY?wqVU>}Z9;-1|LnRz;{T#6LCt2D?3l1% zxjXaqzwF4jP)6N&>weqlnf^``UdH}N?V)~O7CJoPzr-Y;=EAFR5jiHUNRVja)A&S+>KSD12D^AShDIk;pokhW?co`WU8lU$_sj54JUiU&itLh+>g z(Ir3Z6E2x%l1A3@NRn7VKC+yZ>JRAsMg+NXf~7nlmr^e2)3U1T`!i*$?+adg z^OTsaY+9&XovvIx<+$Umy*}{c{(0}}Y(wkxV|U8xXB_X@=F8S*8(Q9Jc&p*&rN3%F zbF=MM<*kQ5Zuql-On2XG_w2w=PTX!klWBNn`ms+*LAmdr$hRK;S@fe9QY|MlRi{$E zQ+KLX%_MI&&R6Z7a%R0%sk-&Iz3a2p4X>Wcx~i}ZrCkj_cQq~6qdCisg5CH1ldqh- z;{q2dR#s=L+fr4l7X`7s>`oaJp_#I^Dc`!(y1l7=y{WxVq@EaFv|Gx)EiPGYZqKsS zVRwMv^;J!t`sMvrVa+olQ(AxARCaiy=i?41!hQ^nap}l^iKeB1ptsAS0)A7r4uFsc zX>Yq$W+LewIot8RQztpGdm7*N7VKG6EZegPE>sp2#5ueN_5T2ivn)9YQCV`JqkGKK)Jv6j9M_G zR#{wOMjhSH^+Ds+o1fHv9XwD;-n1c#hoU)kXv|vCoKB!RX7dN0@TxuL!Lwa^cHr3} zJFnWYJKAV>oUs20%pPK{w^3T2$-~@xdete^ss67NY(bD5?j1=eyQO}p?|{mf1x+Ld zv<9gqi5s}EU&baSLG*x3LrNTY5fV>&57ezHQRw_GK`1c7kX~YBREjDC(58*3F_g&U zAtbjLRIml8omA%~Ugvp`!7?FB81Ya(CEPzGjhw%rz;fa@QaK4D*%p#?@d?`PQQtvY zhm#ZPPR)di!?=eHMu_Y&)OV39rzOCsLE(T;$d~}X5cK`LYy7U4ET>` zy|pue+ukM?ehj*y^1#f_+hvlT;1U{}rykSS$ga8I+{usXKXTuyO4aum>!icl z*UJ`)`V&<2z-stCN~8dG*RW|ogsldj{fIbMvnbF#b>{3BEOL*9d`ZwJUwcV>HIlP4 zv&Q^_SOLo&LXAZ$0`xW$Us%lI3kW3$$5|PiTOPo1wqA7{CGp8_W>s@_7Vy@k$jBHV zm>IX}0Kf$fIXk2nBPwqsP5H|Bt(=u<%rWfDg*YMpf;rX%4II6t+7I5Uz2&^6rs^Lv z=1u)ws-@(d5&f)E0DE>WE!HsD`Tl}La_ml*)%+6zvhF@gag(M4p>ZB6&LIqoF+Lbvy%vVm^jJ{ag;mfed0VU!Y(l@ zpX89~L0|X~a1z=8C8dmNG~i)aB!r>%XZV8l6i#))4vs{L=Of^($0ahQWN3ycDK;$< z*-CQaH)m(&4P1`bZ)N@C0kK{5ykx_u{f_v3@%xqnSr{Y?1WYtabcw=rfgEg_tObnn z@-|}A;A4Mn@ppi|l`DpB)zVv^;G|AO- zNo#w%q*EjBB16s-h5@JoyU#0$U$5=V=Ip11Em!{#&tJFEo2b!D)G#T`99d}Il5XCj zH5<|$MnBD=IKDqr!onO1A#SD1K(?Uu7Gz{2Pk7>Y0G6dk{t65!0hW*e(-gyu)Y7Yd zgc!7F^>nkasel#h`8d-$O=@JJ}N_VJ@g!52c$Qy4`&0=IPX~lj)tO^d9<5 zgdoko`ZEfMPGSan>MX_B-oni7exDv`WvL{xf|(>qzxX%oYbMYkz?NQo(M&7WlmehN z)I^fze3=*B&bweD*EnT+&6BOJ!_9T4CNNVybLO4UTcLErmdqokQqP>5ulW{Ux$o4s zWPSeYzO=997d2a8ZMRUrDP6w_>ceHXJ>WqW-?C4rYeJ?}MSI$}=1$Eidg-*);pGoK z44W2cfoT2+H7pU;wvza_lx{JPY#=Go(J{)BbRW`mPE8_$I2x4qSN+uCv zm3b^Vo4=Dray2qTi0JaY?wf|Vt|{$mns>F(vwPaT;0i!6{?#(?-$nsA@I`WE#EP#{ zW;Y;0OLvzBvgl%Y4UCz8S-M5?+U+MAR-?UJd-wCR{PoZX(J#p3}a&Z0=J2xWsv z@L6nIz*wDRwW+jD&Cq8T4d8x!0wr|ij+;GybmGGk3tc_wuAbD6Be&!~jeZ(Z#3AbcLS#jVGJk=5bg73- zQbe(YtkHpq5@pMeErTLd1;!SF2`RCOajD_^=p|iH35XX6 zor^J?k>No)Q&S*0hrUjhL-`7mjpi9Fe7BD#@(I+z#p%1JcQ1I?-1e@y1GDIL@3qhS z*Jax~Za$oD+?{IJbKP;LZp)nf2SYc8Zcd~-dop!LU{T?#di~V&so6arJ)3SkGVeQj z*IzruReT-3B6XrVOueb1;8_aT%8gNs^)*PbKc--sf>#lk8Q4FdL<&|>qYop($Um#y zwdnENJ+nI(1>ENPKBwFL)fRgbQ?f(QVULS5)%OLuYg%-Q{*A+_tIny^f}M%58eK~g zf&voJqUj=iL1-nRgS64I%Q;Vl`-z1sK#w+jO9d^xfS`zk2i72}D-X1#^j$idltp4} zA<)nOk_j|S`bf|;w%(khC#+tM#2A^MJni8e*4R#xNYX>+4^%LR+oLK)B|_UJP9f zrb>d&pP&krf@4%6tto)c7Z#h{a}V|td}m|t3xI%-5nbP0e&so&91f4cHgqg*=I_jS zivWWGM;9VRzySiB@plF4X{K~GO2Cv`IY$@9NnZgKO0d%;n+aZRr)D`AA3HxBNoZz_ zIIYBl3kv3q<&O-E>!Mj@V1T3yfB*o9>d{U*!CE$a5t0mW9Ax4^A`RurXp}x@T!`^e zJUjsAgddNBO#@DNbxKD8LYhURhoFvZCk7xrmclTBzN~8k;#4P607~3ES{{k5hp;3X z>AxuP=EDTIu*Du`os9E`s@A2;Fvg~$VUqzV3k2Ya=y<0l%HTRjJ`W5oY1j(|6xI3p zGGOyb0u^EjOt7hI$`>s*(v|DrNaRmxmh~^_4Fyk7X@=0vVc$e1Lb%Ez8M%3upb{IU`HE9NZG_1%-hBm71`?RqYGvpd>f59gec#|NODmc zr}}d?YK{pCX?G}+lQk62mjR4H0w3a%Ou#r(KktH&%-JkLC~t90Iv~e)ii>_%!1i@O z^Dn7^6r3#pO^@A1@?hL<%3sj!9t9jUDR)2qjO*6`&BHifV1|;FK+}iH7-$m5R|J|i zT@Gg27)e}VU`b{WK?7ZM!9ar~l||Qt2#vDUq)~>18B%2?pa8hr5qA&=3caw+&?p;+ z<1`ZsqwaBC_iBai)!7@K#+#yr;W?}A&SB}y38@=UO()?JoAcmHCTD=XAsjhmkcS;x zl^}g32$T}x_(eL#Kqx#cT~^LZ=lN+#!|o08+2P@_Sfrm;8#`M<>x~t1DH6s2Xk{()lp3HEiJGSj%x<|4Omhm!`&1_||M)Stec~q90GO!hl{RWn|VdLxuaR2Zy%7RSHjC2aR zV3_dJC~z^`uYi<1#Yjo||2I;i)n-i3ZfG3{Jtb|@7U^8h269hHTAr73PHh6g9vXxm zBkPzlR#$qJ5zye6HJ@v?ZgbY*_~6Q-;fCVHZs4^c!aY&6g+T9D0G;z@lBOwZqPEs_vC@I0#sE{qT(Yjowss`+ciex&FQt z`IfAX29S9Ux97Kgzr1fj3bU2_X>E7CPo=6$@7x)BL&0DHdaMVB0a-I{A0@t^+dT?6 zdZgS1=n-GCYl&o`QJjyG-r*Wf$NDCZ~nA(=%38^B(bjyXf(%gHmCN{mElF|j2? z%O2~E&booBr)2{>{{-bP+D|=UzSPa|Z=i%uFlli;o zRAkW27(EXdJy*_i3UYcrVVqw-jT(CU&4L=*S*-Y5f-4v7Sl_o7h})eHh}#Ra(fI?} zIHm`U&V+5+{=T-&VrSDkwvr%;MyKdfO?FO@fG`;S1C^w*B_`q#ymoa9EwFUV+ zMfr@eFsK6ur~_~BXMcxFUBI{M&k^Kov^%MN1vBK6QK4K^WI2)%W;k*JT)nG-0Mc>buL9E#qP3|Z5!QfJPu~SW>>VhRjV?wR} z+FBAaYEyrTz@#JLnufLcHDqiA4{Ar=MUA?mVQ%NbrhVy6`zAHKFpgq>z19&c_Up9n zcg?Lkv#@1fdW+t=jvCYMNn-w-^SI`;#EEkd9h}%I<`UMe@A-gX46Mcf7p7fHNEYrjP&$W z&(Fz4L)Fc5x2ir2-3q05p82clXXg5`%{$(BeD>i?O~>u(XEOe0Q^9k${papf)W7L} z!@p<|Y9F^OTP>AUaQM)+O)Il5z3q5LIx*EVk20&NDcj$A`KIlUJRf>8)qBtw*PZ6| zbGAS5-0)RW_)lvaGwu84Yxm===T2R7X6?TDIz0T)`^)>aLd~;coDl!V8~ic^EfrreFY#zaSLwWOJ8(u&C}oCUO6{v2 zC)~b5TS$srbg{83)s~g=mKu+edEd4Xcei5OOYCc`?U3V@(k_&;_g%5=C2WIP-coHZ znb*{IgL=ph+=YgGT#uPg{*g-QXHY41YaWrLBiv7t$8(&HbTY%I{HbRMKl2vA0jH+?U?ma@q+YzwCuolOHSd|v57Q}|M*pLz$v!ea`fmZ_GZ+)dTB~;Q= z^DE5@Voh4CNeQ)hE|I_Sl}4VL5~}HordOI4#OkzIof2vm%bZoVDaT@!fP)bXoo~GD z-3ZkXWSmXnq7UM!@3Zgg(_(!}2pHc7(qbSbG?;2!l@?c}gx1AMcRfT;?lRld(Z#w4 zi?yRz&#zFd-$li`Sg}=feErJwl?DI0w13^~rHsEbK7#PMIKKrLV2y~G7wE{G!ZE00ukvYAfYtx%?!oze>UwaYApFu!&? zb>km<&M`I$--!xexe+vuFJD^Cx@z_!;yD*E2J=T%pqpifi<1G#CqRg6PBZmS-XGP&xntDHGT?@7kY#<$cS$5ZIOuY`eKB6WBlRI$*q72y~ zh4$mmxNKT%sS}aEkFy6yF9hw3dv}mxUj29Y$jr9W%GNI`nZZ1HDJl{)YYI8jdM*$% zo0)bUy3NcwBVtOypI5Iuf7k2yk&r*0q^Ff4oR}U4f^gI6Luh46l6nU7nXgT@;UE+G zOBYIVFpZ8OL|~^6OSQXHNzM`_8U2I0($$-+03E@&7f5Web!%FZq{p+DGbH;xedIt# zKW?1#diHMB-yDTCL0%}y*s99Vpki2;u@X5$1c#?YYv8ERpP*ZO-=&R`+#)h3kD5hV z-b;YKa@i)lf{w6;H=QwF2HQGM0RI8XHpwqd3{UfxIVIEF#dwLk4IAgW7B=loZ`!-C z=}3Chk<6xJsg1|~qW;et7EYW^pE#R39!v$lm8yT9i9IcrHt}wLBW_EJZF5~Iu`MG$ zloGbV+I73QXt%XM)dN0d(NiM#PL_LHO4yFvJM`Qo@x8&W{4Tlzag_!UbbeMA2Vpjo zx8uBOano*QUOD)Nu96rUM*Boz>M%w)|FqC9dt>6C0SrFi7fy7b4q5Wju~!KZ*)}v${wm*~09RNwEPUSIm}i)bhGlN75c)iUJ+qsFYIEh)#ZzVwXFQVE#k0G`Im8GYTj72P598=U2gry z{%*JRW49eq7%CMBZS&j4%3B1(ycA@pGbDXsTZG?kn^DMa$2MmpyB*u#tS-}We<89i zpvVX(F^JmSm7SNdDONVzr zvaKL*iIPou?Qa{{n9+wZc)N+>kFjD=d)C!imNv>pRc6?pMQdYZ#yyduD zUC@)_3BYr{Co3G|CHB0;d>~b)GEAi+wZv$BQ1iZ)S95+;+pD(YD{Ny9jzp4Gjv7odcJ8~hjbd&D&`BzzD$A9th9R5b zRz{H-R#%CzKv&p)O@+Ybkzw|bR%Id4{YxsBg3nQRoJ_qzUwixup60Zt`IEBRJ53vJ zuKlB}A8ws*Iy7}M>#texufFYHeW#)(TesnZnmJ{m^I*F3V5Xxxwc$|eQ2)FVGNq(#wm2P}|L=j=;wiwXeTC{qpSE+y3<{6%(3&QBElr7L-Iz>P0olW`9epn+gx5GGPh$V@1>m-obHx}!Z7a9usgM7Ar12kjP#GinfD^-s7h*DDj4#_yb!YCNpyb2^rj25Xm#oQCZ z+}q^kYcuv*Mir1B?KQLKZUt_6VB9_Le=+5Hk-|JV<-O&Sa4h z7@G&_1T%&-z>%4Q;RsLv7$kj|ScoLYpdW|nq3Mw5X6Z76YQW(z>~_fCV~@s;Fhv}< zv*ZShjP5jmlw&YyD4f^pYbV8Y@uo??-wD9gTAuvxMkh{}I-bLxxJVUHA12$&I6G)I4(lOqSD@(-2~s;zp1wSFOHlH%kB-6s z04{Kuy%MwXWc9I6o>bJK@Ms*JX8yp3No*V&A=@2p$2YEExk4#90t?fDh!U0gQCm5} zR*h_~lSzHyanGo8QF$|%S#k(%WY_BL z%(+j{Z@$1XFv@(EB`Xa)grbZE}?LG1bZSiCh6Cb@M{OCSKbZ2 z@ESk^KOg~g5RIJA*>RM8U_kvXO0-jVM{x{>yzP)ebnU?K05C2i`hrOAuo{ay<2o2J|T!QKV$>a=(D?2dUj=J3^VMtw$}_qF3`rfvF(>Bq8Vm9Gb&WtOMH?p#a%7Vc)LD&mIxeu9js z3|@};?lam0*&_~&j-zN8Ll`2&Tp78EA@G5eUV{;}oe*{_1>Zq-c7|0mpirABwS|Ir z=@p&b<(ZzQMNpnoe}8u=f@dn+~?GL;}0EV`Jv0CE*y z_==z75ZT#L@C)BXmSm-eW(2mUV=*!t#}<;THhkcN&u3d!tP?!l_mG{h7B=wDYs(W+O8=J1T%p~6V&RgG^8J^odH#8T#siyq9 zQ?A`?wLz9kY{O?H8~u!cr;i^+a~F$5Y2lYV-Z)J!UPWNmpAnR*6v)))9e4}j=-sEE zS+qLsEqC2zpWAW2UuCgBs-0-G+xL*}3^%4bTZYTOJ)Fj*Qu>=_HbhaK5v& z8z$H{!WBc%t~g|i;)G$W!-V}^4Ewu*{arb)X4iHaT)A1TW-Jd>NdU>V0_lYPl@6kIp2Zr(RH_6INBcxjUJLck91;=O4Hr32r|RjjAo54t^( zfs%fc{ieqL`u!$tM4XJUSiiY%jN8ozAzSSHuv6iJNH>jJNwQkxM0-Mu; z&G!XMLua<8H6`sx*L2;t;^{647Pq8pwk+BOk+ZT%8$q)Ae^Tqr>Rg5ZNsg%fJ|9gt zem;-?d?2UKi0x!vG&s!szw}7*E!Gv?%GIYc6ReXD1LLTL8TrR1YVE~h!`7NoAU;7o z{yo(8rI~0WI;{adH3BoW@3g+vn)ZQYub;8Mfb z^T5ukrGsV75-IqaSHF=fkW!912mv!BOgm2zJr1hjhd^#RTmP%sDuMDasBcuEU0=_Q zhj!SA*_b>+Q}!qY{|set!K4PEj9SDGk1_*l+Ho|G=0B#n*^1n{3VynL-W4D}9_7>J z3$B*5t7YERif8T%als{}UDD58{8!uxw2B7#+=m7b)yOC+y)&kJsYRz&Lsz~6jmGFR z=B=5uL@SKdH=x{Br1DB=s7mMMzN1&{ABl#GF0~cyCVh6%^$PXdR{C9Gzdi8P|M-)3pK^{n&=#gQ8dx#atu^B;GZ3Y+3cged1 zFK;3{`7WuRa?r16Fq6d%xACKZzN#5NY{0!Yyt7wt9!k|clJ-3^b@Z;Q{CeM;!8d{n zwd>Ng>rypjCtiM|d||`B^oD(@_4_|cEF5?;ec;K31J9-pJe%78T&n79+I7}cx^&ws z&G!BNxp&W{*6v-Y8pCx4#PhG9iSV_3jJkN7f)f-}QSi;EOVW3#6nuwzz83`$b=i03 zo^_SIDSPn9BHbC4iTm}|OSFl7V=@!| z)pYz5bQ1TOxL5r5QDb`2KDF!DNV*;=Z6oyCUO^Z>-Bke!rT z&!vQ+c}+Pm4m{TppGbr=J+Zr`Ak}Jq${#t?`w5rT-8KQ_sO{JEokmrcK6kTPTdQ9| zSN*G}LD+tU&9ww1wdw#BLi0>S5`3n`I`Z{I*m;wT2 zU>nA)di6|1BmHN3`(G&duM{vaB(YFVjBt4mk?w7Z(*7tVIUjE76n{Da#9k=~E(ykq*7gv6=@8-62d)IA$SK8He zSO2Akhqg0u(WjMl*T4Pa_L+;B#*LZEP4@-2yL_sfyR}dIRxS9}g3`EC(F6i*R-EmA z*PgCf$Kr4FW^0;ex@X0=j!?9D=Be4rx6Zur1eC%{UPoos=T=u)`8_L&dr;a$Ak7w_ zj7SktfsOF%5ETGxAT-((znWBdtp>nHKfF|bn!7KDKZpo^a}B?<%2ajnUN982S~<^s zkfv0VKiFu#`Fv4hjhSNPM7cgwT>kX`&^&EHE|3!1JaJ;8Ca_Sm4r2x%LhiW6I*9hK6=C5@U_Rb^Y39KG$Xm3lgobM>LJS0gP_P%j&19H zecH4w{h!mky@+B+RJ1=;bpXHnRS((ELfejX+m1|q7q}DmhUsHd-BXFIuln`g>E4;= zX2Y|mXZFwga7=-xbiK3Ztvx^Yt%u8N_XaZIsCj*SdVHon<8PboN&7dXTpRf2w-?O< z#`qdLGqtv~Gpm11Zz#A)ZO4SOP5E@+J!`eSJbR#Lk?w3$#(jCC-QEaG4En_=yS>AZ z`E>C1P%-pl#KSn6j31TcKZUfH+=%Y?5PI5sencMEqF6ROhF`Icj!4grs8P**K9SVY zcdl}GqI906@u-?eRhNB&_u9{AzKajDyg z9ZI^jV~j{NqK9w*hXf@2_m+u0&-2AA{?u~OFYiLy`Jd>+qCJ5Cly3|N@s>O?bs{C{ z!|9(`II?wcG_j?tW51f9PswR5JKCudR^Y$COdLnfi}M6w_A@?8od04tty!GNgm;_E zYIu}+xX(G@b(xNX5}S~7X@^7O+*+zhJ8?faK8#;HhsgmqCFmq40V?)=A*Xu=Db6tz z861kjk5ZIbROS3|9uBqHl4l-&8C*ZaB(CN_e z6Q}#0c=~M4*?qY0#1p-zbFNcIPCfDTi6e;h9e?7?vEzNG*>vaZqtMf1xX zE(CA`i}Ts|De(R3x6lUs7vdiS$``GoDE>n5{6Z*a|NZ|~X!w^x$0tJ5Cqm;VLc=G* zu1|ymU)XG-1wP3{%jX_J-2I8L4v5I&{ld~AwlBh}kbYy@;$5W2d(Vl2zvjMB>9_oK zS>>V?_kV@67Ixh4IfOv-qLX4Sp{jn-O)-y96Ik?8%qKLgS}dbjxlmcR=%-kP5NKJf zq!>;`)hl>E^+?Jmf?V@LDcv--0xxy(z|FA6Jwp`;C zZP%|Z3%D&e2Sg9Waa(@aCiaQf_u+T{a9=JzCW?XUjmvOsCkB@LEUkFjvMk`X*dw-! zftfRl0&c1Gk1o-D`J`Bdc27OOEYN*$6Q87MixEo3`E*GfmsGjT^FU z?f2Yvw+CC-ukdys|F)^R+A;a~;s&9jWo9^4z8OEZSkXB1SjxYC@_4qQY35wY-!Xao zU->Gtm9^QLK(@9aTi=vzmMGrTnys#<^s3e@L~r*@KQ5O!-JUPX1#ji_y3Z@!_|1z5y35JD{|_wA7g_)S diff --git a/demo_agent/agents/__pycache__/generic_agent.cpython-312.pyc b/demo_agent/agents/__pycache__/generic_agent.cpython-312.pyc deleted file mode 100644 index 2ba36e54213c928afea3fec65f30b48e9e4978f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7576 zcmdrxTWlLwc6T^KayT4{q$G=^9u}>aO(~Xa#Sd9Yw27ax5Nng#>Dt`5vm?$(qRb(c znNe&B)K+NSE?u~=yuiYCcZ(=Zf!MHtx*tW;uWq|2Hj6D#O{9d}sf@U2`jJ1brJ`xl zkDhynLsDA0K+(TmL3i#s_s*Gn&pmJc(&yt4wC}F{TK?rOg#Hx+c5{{noBs%dJ4iwj zl|l*?rzkvjq#Ozzrxhm7D9*T(;OLYq?t(cc`L*9FYZ(7 z;&lXer~FDF9w1{j6;$fu^H^h)xiIG56;WI~V8rc2cD$n;2} zE+$fbjg3#A~`L%YvQ}G+j08ZM2k8N;p`|kIpG# z`kbt3;*@Oq&d+PQtdu~Y>3nTUPU|oTO7m$^NhX9@HKWYx$@CN;xmk5CEeq4Sk^)lP z8=|Vor&Kkgnod>L)p=OrJ25Tl=Q5I79^P=1_t`iv`vCF~h`WyB zb--e;88{=SWi^=~N5s1``D%>G0Y}oh0LPZcO@9K;At?ADg1GY}_+s&CHhZDFgY?qQ z5Sl4ndG)^R>pzbkP~$O&>Vrwsc{(LdX%EmQUitOtz+0NEY6A*fa8k+iC&WY|DIOe< z97tcK&Pj<&>yQNaDysr21_5wEgx}1AI~o zK(~a}Ph7~Kh#M#3YbV}bLTU>j;ST*JArvOXgq~67LFVvozhq$A^eEyrf!ubYd~CpD z-6SlD!=Rca*_uoNET*=j0oJtx1>hsf)|K-p<`|FBD!zTS9d~=6i!o{=9&g7!00q(H z)TD-Y8C6@MH#tSTB3sN_X!#BPR3n|6c#E$P{E=hlg8TV7X&e#&gi0^ z%%nA#BD)X5yoy<1oeqnN8o?btr%5N%y4sDwX58W1BIi=^jgrm}0}YG)AE2jAT}xxj z^cvGu=#DLYd-?Jj6Z=(nukq5`>)n@^POW-d^WF}_+p)%6F6?SrI-Aqhn5IpJc3KA_ zONxbBcX$jPLhS+bivfN3G?3#3e8av}|$ps3@NDzWRi0H8ZHA$=JL>={Wf;V~ zuJVRGn!zf$qp3$~tHcbx@g2?nvev3&F(XZ)Y^w=w3ieRzy8>&khWU^^t~@35h^|n{ zGOE9dK9gmBjQ-S7nV(1VDCQhrNORu-?*wjRI69}vf|dZ!l-8y*dNip;HCd19ndk*| zPJSUeF$ci39K{AVN(^pP%E($2lq0Im&CX_2J^F4^12yT7o=j@uL<*?xAliQ#%oeD- zEG?WWe|H=R%xQSBn4XVjCh=D^ISST%P60!&MXyfF>1Z;Y)RUlvung~l7}nuvj90NP zntoywD!#yE$vBl#ESbc5Ni4dhrr0K#b=wbYa?_$FSSL`i3}9yo9Eds9F8ua(vt_!j zfLou^G;C0#xLKwPtP9?$AS~>vHeCHSx($|VkD*&a#X7{dta>{(cK7`_llyidyf+^{ zYJ`ujg^%T!o4!JOe~wvaTMLm60PCK51G!f=BE9*@0V8r?Ei#xp`Nv~VBfE1aZ;n0n z`fm;27|wgUKJj)H8bbMoy+*^{e8Zs8Fu2}uXd}>AbkMtwY|+kOeUV4|2bY;U+-+_> z5G%Tn$9Idn!QEwY+&bI#FW%q=A6RX;xOz!gz4*P=p!g|2QKS)HPs~`NtbK3-%p>AT zIN-q#I0*DWd*7whU`4O9l;ps6(Uv=aDQxyYf~zS32Dcf^%N;FSr5SozGiS=WP+4A^ zCSYqAI>6Sj34pDk?|`kL@t|v^-To0*d&NwBWpDNfR$>>pqt%L%p0hH%<5d7B=|xA@ zaT4946OJjAa9jp1Ez((fiMr{^(zhKqJq%iO0LGzub?l2s%a(0H8lt;Jri!=N1@CA# zxGeLW9qrz+6u?J}9SAHsb*y=$Rehnu##NrSRh+@Pwxd0A+VZ3CwWqSq3Ur>MOGiMS7ggEt;8;XG)bNX@GV(Z z;!3hyL&9pxGAntjwSmHVP(u!DzdLix-cRKz)k&zFzJOoFjZm#2$Nvke{+d)>RWb(j zv5d_OTi>kiLE-`o$RW(LQUG&Ef5o1!EkpM0!ulZLXo*i*cG7VkC{cvkRn&>}12&~D zq#j@NNL~pgDY&Z_ZG6^KMU5w4KWxv<9I;20r!9BzZC~j&Ye`1wt}j6XyXg78=jJ!o z^D1=}U866dt5ju|vHEe-OA;`NWMHP7BzD)pk(l1hgeFWUH4?46A!^4#rO7IgFsMlx zM*4JSZfaWBNL&fITM$R_R%!>i;T3Sr5FC?a2S=*lOx=l08gfc~Tab?J9{cH|Uv?d{oYk?Xb-)9C=9&iJqj-V?LLyGw?aQ=z!Fm(#zJlVml_m<`AAZz7IDA zJb`D0CoyLH^aT?^e(V!waSjThw02ciRUH1RSV?1y$>{Pm-DF`#ASpB@n_fjsrc0;R zus(u!U9#M(i_lRQ_HJ{oQ?f2t%!K4AlMqwdNWZmEgtxWVq5BH_t;6Z-E41|FTMij5 zhaO*8YZ+c;Ph&!_tQY{ zH$^v0e63*{{`N%bY%VS%Z2Y*{T0#^lsQt-E~(1yQrV^81x6OTO)#@F_|lsmEPGx*(4Sw7E( z4K|!-+YPpTgxEsBJC|=m+GaiKo9^`9 z?%ndbLR`*UY(&1`t+5+pd48Y4?Pa;~j9^eWT!G~9x*7SKcbAzV;{X~bd2Oi zmoFK7d$Aq|m;4Q2S&PB8+#OxHXtegPvjYW|zva8(%d@Qp+j{rndvD)+`~JSiC)e7J zt+T_mCigx*wAMDX&K`R@bn3IN(~l3Xy!qj!4=#Pubvnl`!$s`b=y~D(h5Y{4jQy{z z^&HQ!tHBO~?R@GF-DM5`o|U0hf8Q$G_n*(cL!scIueOFzp!pecc@Bb-NA|3At%mmI zMmIXTSHur`N}J#48OZkx8$H8oJud_5vcY#2I(k0rf4?6u_>6D=rsx9#|Me`4y3bG= z4l{m6yGQpkKl3us|AXu3=pg#};f689d_3qm`wIQ>OG9Ue=wA#uVZ1Gl#`b~4(PJ>c znK9&`FtXYd*l3|_xd~LZ%>=@4l189X^nxI}Q*o>a$xA;xEOS5%1F;F%fIx|>3s2#B z4bv>8Fe>(?3-#Odgd{>scwrB8pdpQ6$Vohfa5KqNLJ@fIHq_FVV?JfWR^8)wV7=*r zI@Fw+-gat;Qms>e2B+Irh=LpeM@bMXlurs^sXeD0Rcj^of;*bzxU%oLlxwZ0|A+N8 zauA4oOI{MZsS~rhQiTAF>;J{MNcC37H80ZeN%6VtNCIFMpB(B7o^g}I!B1vlMuzIR zD*=IuteW&h<{Ef@{%TU6CJFaMN;c~WWUUYqnT#rFW|-h6GIMDO@_qqcvu7G6lPMr& zzb$0asd;q>zH0hP)%DUhIlI^nHTR=1A9Jf7co45>HbCeBu2(OU(ZwmV&PJ9t(PqQA zGyppll2pvYJD>6W>p#dWCA=na6;Q&(d^$o`2+m&-$Mr#%}8H? z_b>AX-&TmUz1MuNIUhM{M2;5PJKtNlw~%iiGTMhe8a3KaLgDbk`1|pE_lVIw^3!gk z`%EF)^&$5@myeDb(NXX?&0LPooxM@F6+mr+n5rLK&HtHd8>afY!A5}W?tS;^`)422 zt+gEmqs{ zgNA?b@u9yS{>$Mt|480{&hVdGWzP|g#OU#HH3~~D`&_4Di>mI$9hfFW%2`#Fsy>Xv z8H|cUSz?<_1_#ROI{-tOOH8Gp)F!KQc)A&Pm3lKNx8trA{xuo8>j*qvZ+{`SzYuBu zwI}LKP){R0MJFD>Okhvm-(&cDif)2nQDbP6EP8g5`j+#B&EpQIb9b>W=p0(=U9-w?jT6>SD^v4I|D2voY5kx@D$zTv%4Q%Xrl%lSPNw^{#I&r4Ryaz+Rj>5} zO#g%kzs=xO8^Y!(TvW7Vq@grzg|PymPL$d7TVH|#VhKxf798S2-H>d8gy-UH(z-F6 zZdpHXY`+8erN{b>VMP5sSc26-`z`2-G(}NQQ0M39fPoJDGYULGkvK t^#t{Pj`lx6``}g_-v4lTDEJivx=k9uA_lj@ze`c$)binfM`S>_{@<%IaR2}S diff --git a/demo_agent/agents/__pycache__/prompt_utils.cpython-312.pyc b/demo_agent/agents/__pycache__/prompt_utils.cpython-312.pyc deleted file mode 100644 index 0f30eece576311d4c075443d428ec6f8a83f4c9b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1370 zcmah|O>7fK6rTODalGExQQAa70R<|WEK*{TRw*F!t5g*s2}lG9MIdYK*}$6hdNngP z7#!pS2P{AaBotQ_A~%HGOVvm%C$1n4Buy)#fkPyE;g&WxPJL5$V}t}eY2W+iz3=^v z$Io;P>eMZrl2QaZ*k#p@WR)FxP5+WTC>j*q88&s0ejqM-F zj`-P~t*ijOmwo6X*=;36r5nfDmq8Bjx&qv?!6rC*egKIK8+bjfc!01LFoAO!IO0)! zAXM!7>1i9l`1EZ&R1w_$mJdq9G1z$)RnQ4!#3nDvr(G`fU5cf)4GszG#R<@A$3KR-DmdisXV}$a^Ig_RZGhG?$gLfX z&IVrAzN(J??R%2*!}Dxm__x$AsXw)Y!r%Tv>s``%8&mTm%i7NB=&N{lL%py6sQ(mS zjt~5cVxcY9lvS<0zPq-2vAcgs>tBh*>tnUCo8zl`qJFA&YO!bUlD=1XCu$Qnk8OB| z|ImjQ=Fk_I|p@<2{`wUX^%Sk`9IJKx)KR2E3S94P5tnG59R=Eo2*kx8#z$sGM#ds$APWGPi7Dj( diff --git a/demo_agent/utils/__pycache__/chat_api.cpython-311.pyc b/demo_agent/utils/__pycache__/chat_api.cpython-311.pyc deleted file mode 100644 index ef9f84e01bbce21a3554f3bbd9645ba088cc0af7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13508 zcmb_jTWlLwdLEKPig!w)sJrB`e32MSloe-v+g&@hWI2wnjpa?k-7GWWj3nB;C^N&z z;!<1fB0#l;focI4@ouwN_a#jb7j=Oa$O3&xx9vM=unjSQ0i(qN%^MA8+eOfae*YPA z$f4wPi*`IbI&<#V|Ns8)9REci;N|fA>d{mApQ0T1Z}ic4oORE$J2sAcpOZMrmgX|H zxGiIk+imo1Pdnlc_LqE#hje&xHqT!c%9z;I0bKEER8IO&V?RM@wC-uI^NqyG1 z50T`PMnWzgW$F5nZa^OX(1AGoj3@qrH25ADKP1_&bJ3wMC@^Z*U7{+blA7+3L`_Vl zMOD4>1tyE4g4dK>W>s5|GplJ)llAbjtR+@iSVF4@MtR+LR#fH7vZ{*9vL3j;t|G4S z>3=1k5wndi&zTE#Z`7uDtcr@364OzK?!LOJrE*y@t@D>ss-`>6WY>|xxsYE?%hZrp z^2^Jq?DD*rlwZj&>fx5}YpGQ^oyy8c@10!{wX3Ug_RNK-O>t8vYT*t! z(+KLH71QZOF?s8|MP0})$%>p!%CqT| zoYjy#a3-(i7IL@bY^oqD2)RnjAanCoRC-LSW6LkC2f>2Ddg&F)lwG4?^491Ajg(t z!n-L^SU7(nCR|t&uH>@vB-NEQDl=M#W*BW{jmhLy%`ARV7SvTanOaK8lF=ktOeHj0 zbF5F#Y03%(1QyvUod@ zmDh}3sVFFwwHA{RZ>KW(jF8P|7G;GEHMN1YmaPrQC!?s9rNl}OB1l3hi|Cv=j<2uC zs8wm1=9DVblVTuRL<;6Y58npUgyrgAt*A5T1gPGLs-+ZiRIP&ZbQ->iu2cQuz{d8YnjKH`W!8% z5}3;6-8KQ1rC`L>7rmu`G5JVxQ8_BEt!8uYOe?Y~i%N23T9a>UQ}tm-|EADaLoS}2 ziOs|gj~UfMSYx2tcioxUbM+NzZR?mSEAOU~GM3(yn#`+H$X+jKEvMX4S9dS28Mj;e zZ(K;sUcGkV($$5;_1PQO6R%%=?c()oXhL(@X*o?y|L+zaG8kJx6(&;IRZQ(60TT>Q z)Y_+)4xcz`)KuunK`)7^>E%^z>PYMxQ`j(zIo;b7P*-%1@c~h?{PUmD^Q+UkuVw$} zLF|H-ZK8M5cWe8N5Z0u6kLGocxgm6(*5ntY%=BP3p~#xDo}f{el_Y8W#zSg{e$VFM zyw4T6n~fOmN4@wm(<)tB1K=TrWcv^?pO~>fhO)X2hzHSczfyR~?3|Dkv)P;`w8}MM zjpU)(QUN=2LCMQehq4zXD+l4|-tUO%ynIekathQlugd8ql`0dyIy-#>-B71VTBS0% zSP~K|CC*Gs^1IV#mE0O4&#!0jnaL%v3$ogDUQ4CbX;LN$aWxfNUDpGN#45x_B9%>P zi9}(@qN-w+$X8IQ8iTXRRehm5r%Jv9AMJk-!=K09!=;hy zXUM@>2AXg)8)(46wt>EH4kP7iJnciQoM@e#`zBs&q^G}<-8wDbL#hqihF$AurYhP| z4y_CJz;90L1BVquUgxwxBOPa18pvgq_`K2(PNxd5BLv}{u46~ zmXh;|?lQL0SGH-a4*_YFF*w*uY&Vcrh`Nn%S~o0G*+;(pR!MN zMN`*O+KTQ}SJNp?2~$EUnPt{{S`>u2IM~YEwOofXg{-M!9K>+SUCz5|bZ2kx(webW`+bdBrs1gpcs2UkBIo_RDp z^Kf5z_-JML=ua+ho!Kfp4v$oa4j{TRJXsyuQyGfYI{2R8Qx49TzP3G z2dArpqaVcT$@lE5?0K=e_dsRu(YnhbS#Yh(L#4p^lAIdn2?UkHXrpC;Q?JpYgoQDMIV_@3-V7`#_@rNS4<8SR9Tk$`QmWw0xs>HuB0*3Ydgr2n%BdIt^*0 zuH+Pr0kFbo+Y}JQOf=))rUG7sv&lX21-JL#oi6$I;;n}J?wpa}(V zL&#@S@8o6Smb~6TTdGWuBAZq@A=u}2a^x?mqN$Np<|WMv6B1P&LMyhGKQcBo}DX$ z=?vZVZb}8bfGsB=)s7~4S41^oBq{V;Th@%Zg`8>#4kj+tckiVSPM1P2l-w_{CPfEy zcY@7-BB6T|39O2IT87`BNW7C5nZ<^$jzmJrB@@P0)*V1JdMJ@#T8!a46;Vv0FgfI8 zAlMKZ4&c}p@xb&LPpDKO%U|aSMJgnRlxcEKP@o$YHXyaOu6&ciUM6Q2&c_^UGh4vQ z9ttIA7(X?H6|~v%uf~r$-8GJ!mu+0%)aI+(juJnKx6DsgBLkZkw$u_ofVa#KR0oDP zFK$Ol{4m}!KMWDDd1ZUH#E;@F^P`XkCBA1nS>o|mDD%S3fZidLsFnC3yk&l<+7;QH z+uC2^QD~&hN2-HEMv3+K8gF;ru+_L0*WK-e5HXzz&K0cLdFMWeC+EJW4t&1%h7sVm zJC9S=y5qJx57B44V~iDzP!d1$>8|^peG5bOM_74QqZ*_PFWI5=5ZgpDahTb1hs2ZN z%ThQc*Mko0286*)X_$#!PvhB<#v{WJZ^|#EXJ$7`@00vH();BA($vjwIVgqXkkl!6 zN?me5>c%gOUk}0~_yy#^Quh|OWCKa$k9T=E4=0UEeR8)PZq_LEqXpe^uo=?yfiUvQ zJuTmZF#U(*;F4Dwe$NTwYy_r!?-qC47VpFU+@l=BaESkjkz#hrENs?XMgWl_&|X{w z!qdoD2ihZRPF9u(z>|Wtm|9E_t94_3VFkdCaZmyn8UVM3^b(SaTsK*lAl_geP+iG- zJ$aQ#0;8n5MaP=7)2b6-XNy8Qmjr@OuE@z-t2wj{cDW{C*S(vPWK@}!Z>JVhfB@@g zxFE@}(ojAtk<8W>M9!KAt7tWsg?v>-5KWA($V!Hxx5H?Kja#=ISaB$xN^F#{gVH$3 zQW0N~PG^J(V#Q7}>Y@dUz*eoZ76xaMtlPnqtBtEOOQ>^q=I9#f$SeaWhZQNAF>Fv_ z@f&82DnqM){;JzFdl%67uWbtSj1Oa)R8ovI(H-c=mBJY1V;lmJW>hWev{ngXtOFxY zjD+$BSZBJ&OkX&&@PBQ;9%>sIWdd~+PEos>W;N&!EC;lXSxRiC3~Nb9EeR>`R3eNB zBc4`(WC(HrB^j{L#EvZ*U@z;7N#G<+*o$z2&kp>ax#0e(%?b>x^xE!N$(qK-6+J~R zL7J95MSgRn=q$PlsB?pFh5t?HFWHOypK+4o7i|*I#%U%f=eV0Lv&=S|5A~TwfY#MW z#Xx;Cydi@TZIz+Qk>|9EFWS&5=gw9UqHL)#(o*BjmYF>$IwTk20!&oTb876cY5-98 z?xG`Q+wc}Uel%;z<8Ra{dD`=6Lyb6$Ku6L2z-#UAhEF5tRDX)Tn}k!?Q*;#h2R^Im zg`Rdc;w2lfuU(mWPD}jlEh+Lvuc1e@eT_2ECR@?_AZTq8^WqW?kIT26cI(H`-yuz*c5cmdWq*%5?HdKX-GPft<{vU6vCvPLC8)3<<*5Pqhuils92VwRdtw6 zqx@}BMVPo|IY4Rhg(&$9F=M1HNS_er#{?k4@=1Vr7J>SV5)FAvHPF+fNhSzNT1x$Y zA~}tU1dFblV7S&&+x3RVYte~T2JA5SK+6zWDJ7h zUgn~V3D;$f9c@65U}+V`g$aYcXdxq~1PW?p^P;f=&JT=35Odj}Y=%+lTDve26JC?m zT>4$MhKyyqA}Z1pk@o;5_2s6e#0xrm>3YpaM3MgxF z6||+n&b7=j^+(I)0HoH8RZH0o{1s!ANn!9(PLvvpm@wfIb^@T+jn^*GnHNi|JDE<0 zp>Ar*!q|0UfoWG@z9`>L3}+BM#`++bpvbGaL`u^A#_qkfM$~zsue|{b^rhDep*Heb z=W!^Ki#l~)6V+SF8<0)97tA53=tM?bHJGFCAynyu6YT*#l!tjnhNHn1GPSRFq;u?u zQtv?hgcp@TDk(($#+fO&RuN>sq%5%VVb`()uB5Dxj!;NBDmTa>@u(08S4gqwthI?I z)vmloQKVUwesV~cMuUuNGK^$0F`MJrhCGz-P%N`IsYkji$<8r#p3XXTPa?svcOsz% z5!;Xf?C+*wc8UlujI=;L)9eOCY7EC6BvJnnR>AK!x!=Y(Pw*!lcRD_I_djy?m)(OE z_h89A_+)f)i?6tcp7;icLm8PW9e%ky^77V|$Nuo0tJTi+&tj#{b>n^Num?ag!GW0v zs>;m`j#mbbJ}i_6&Q=D_Zt-`5kA0DC>2CI)4<0M~j#qrJ3|*dC+i#4rmLHxhbuJrk zTUj+1H#)X8cjwCEeN^7Qvz2{kx8^E-;fb&JUe6=nNXa)+Z_!)rS-o6f@ZL9T4tu|M z%U^SI{z%CuP~uCafp3)kM=SoLCI8XKk^LW?`1C?4GGC6&S0eLU^N;&RtDU`-&Vvsx zmacuD@`FZ$gC6z=(Uw=X@DB|O_SLw6=VZOaZ5a+bq_wF=f<%dP4@C7;FKU}xvL%h+(b3u_$Y zQ_cOa`#Q7YsulV zBApF)(S4oU$7v>}K z|4oV=-`-#BK($&6Vs6szlF!;!%mS4P;(w##O`Pa#cz)=~x{beUwl(gy<4tbOcIOYH z{wswWq-fu!W%Bl<@OH~W!KVnj_-$jkyxn9Y;gf~bayF+xJ28tG+J)WOAhMF7w%9xj zNw;YM(qJqwlDBYZ9!3~xwH&SpXizW>pkOE*(jtb}|(V_KrI{Bfq1V0UH0P>bnxgTZg;QzpLkPUJB#T>YX9 z6WF@v-0h^yOkhJ>@MK#;lcZ&NHR?6QH! zZ$bJ!33OEgduyBx`H->_iGChA`6zPo)8pmHTqQDx6v5Dzqh|Me25NR+U;u{_zdZi) zqrV>XMJ0{?_eXlqdbwYFy|Yt}U-z5|%}zM} zVZw{htMpTk?`&s8IX+c&D6J=LD$HK(K3 z^^}8yiS2a}BaX>+x|oA-_}tvg(T7f!{B?rYue*trAI>d8|zkKn|Pg7c)icl!sG?tL{U zg}As--?nyd_U?KmxVP3pL1^7*Y4m6%a?Hr&<-)x+AI11NZ=e<+9~IQ%%-L+t5$vb0 zaSYC$X4|qOXr;0p!DHsZis7^INH&dWM&nzuYkVUH2j7mAW5c%0ZQyWwj(aP);V9bf zQ#m^x&C?7jwlQB;O}N1WyNnj?_f53W%n4KgbkpRsc8qjwv3XHGLY9<LlHas_TsUbvoj1p&gsiq`|<1h)POUCqd6^ z@^Vf|70Bq4#pH^aeG&9Gu7fGLG)!Ao*SXecC~5gN?xoRXC6o{1m=XPSZ7+$dd`qz0 z*dJ2COF3fmnOO(%ge#DiOQMtDkeio8Ck6A8s9{kPpjgUhS)*YQucF@8cA#!*W-_N3 zx{z6&)M#O1TxEAr$0vnx5X=hwr5hsbX5o00!4X5PVvMxLFqy-%-k{?xG*Lq#3llWl zjK_J-xYx4-asU>$5KWYv=$}S5vqH=VS=vBDtb=CwIzSC9NR^ImF^`SuB{T)Jp~0?H zg>mZfIP20l1tRS@9sf4?uGhqrih(z&nC4xYlXNK&QPl)RN>pgcP%qKF=l*XjO+ZJfx*V2*14Y;dj&cmx0k}r zRe(Uy40MciK`^H8^jP#QvnN#XbCFu(rmY$^(|_0W5mx%a7!BOmNny-bD`S(wSbh8` z@Hx%in0!tlw9MLysQ2i(S)-%{ouj>K?Nsup&HlOYf+6r4YDm z+&C84j240h8`YR^T$dpUs1Sfu$X?SO0Mj5uF#Wh6P85)&Y*FMPI8moUP*v}#uM!41 z2pY30c{-HSJL+*DWsHhhq)JkgAyh-Jt5hU8U*V_z3=SxR7WmOWbxRu9vDm~N!UlS|&MT6oZii{Cpx)0T*lCzAe*ZB)q=C2x892oRt z>x5z6!<2#UP<`e87TSr+8mAPx83ka#sWX5A0Pi+im5Y{kd#jvhll`l5zD@S8%AG0g z_Ex!TrT^)zaw8?{TjdUxp8r<4;3oT5<@!t3*KE%d?v>JRZGx*QL?^O?r6#S zR=Jl;*0;(HmaK1;+gq}}yM-Joah7*yPpjMuCF={^XzSVRsBqzX;~z|xrZ1MbOJ(j- zg}Vd~BJIAK{hG~&iyOPUPg(ev%*725yYD@ige>Wj{k+`HU6M;JF5X>; zA_As##%ZaJGp1&ek=!I>_f4rZ6V0>__EX(-`qUTDWQNE_8f&JWY5fLSYLapL(EmSs z0YFi9Gwlv}aIXLP&+Wf_=lp9!gO9-X;m41Je~A(DZT`Atw~ zZ(=WvMNccXV~dWs=}%|=cWzU3{#x0<nsYc?yVIW+}6S)Z|LPjZN7{_s@K@23hR4SX#sPUBaVhU!yAFBXeofka+ zg3PIT8K-zw%5qiCaguAx^=YMySaWSmGIuSR5f=4PDL_cf)RReb*Ti%_&1UlH89}B~ zjXY3p>D~ajq?4MTP0nXQ^zcAs1_o!%Jsrih#t|w1_o0SC=u{_Jc z38AcKl5@HIPE%5yiqjV_(}}=6Tepq}jWwDRldzO?I~@YpmaJZ8Z2CF^eZmW=8Tk;m zn9F3}7?uS^;N;Z&uqs?rhm2{5@eRRPb$NL7VEkbGK%d?%2x*w8?Wb;Z?3A%1>v`={ z1o^6%5@72MDXF|N1kDXXi&^=KlG~YFwQMK;FHa=LE?k^AcVRj?HFkL_`RavBXQwU# z3)O9>W>rr5^EMtd(2sz^4v3i?EbV?478oM2`Z2wD;H5))Ple_z=p#-Xp3A92FT}q# z1P5j&tNE$|YEtv+2@oZ7fAcPWA0E^EHRneQ!YQaZCR!NNwfyTjlqF+Cr_^}uJ8MU zPZ?^Pm`BgYMW==GZj9~0U@r!Amx9|vVgXuH^MM&7tIDFN&TCF3CyA;Y!HP(kHrASJ z6pyvp=uY2}u3a92rW%u^^cCD3V9v;K%nYTuD?*a)W_rpB-4;PKxZ>N2@pvomL7?y7 zW7m&u`S-5-_r5>B=^rk4vF}}Y_rg}!!S$|#_jYe|9lCygb#%3GKhj<9+zX{!U4!dg zgXPX$Tb=Rs&iE6zqd8b1j)vfq0BIh8{P&K%d#v2i^IrVjc(wYj-CMh!U*Gk7xp(hY z@1gbHL)C~;)vWQP(TfDV4MF(;@K|?naMbw!ScU)`Nd^`eV%xu~MDE$SU z5Uh}g{@~i4o5MGT*ZsZaNbAjGH;!#Z2G%13w<8;oq1)Vg6`bBc^iJvCQ$s99*NpB~7XgStv91S{mzP~@G$ z^};Xqz2E%v!Fzi)Lx)SA!%q(zgquF>9*BI|?tmfL;MiXk6)IVGuA zDYVsdS1q{$EtN(HVA(>gw_e1(UkW`}@;pb`i?wT>BwhbxQu8H~uq*PC0P#RF`9_|j z9wDUM$t0gmCH14N*}-zsLdhi6lGKh=pomHsDwmwp*w%%J9qi>AaY6N@1Fc`VADGe{ zXsya9m*im#UcyWdc<^Alt4s3NG4D7AV-Wm`P@d@qmUm$;23_#4yaU0H$sJR%+q+iIu0=N-U2ybQCfCL`96fOQOOEEX)P{qtjka_` z8+F6cS#FH3oLt?r;fR(yI`uZ4pF3>M%gpDVCTFlRKwQB`HivWfV>`qu82db6a~^r@ zu{rnaydsAw__sb&H9U5~rvEG01@%rdz%35m25QVW5(#ji1vl?NmzI_}dDmUHx%I(` z#!~7w8>QZwQm+7RHA)PqL)~Xu$Il09>I8%aC^Z~tA;^b>5FZx8e5267H^F}d{+l5` z3jYm4!)((knPmXd2_zc5#7p=dzEx-vBG?Px2K_Y&L5$OMJEWUwtOK0%P9Zq!lv`cRCKSwE<^uIkY~~4 zM%7j498r;`81DD^T6xiIUc_DQrmj&LylU7;SzgIZC=N)E^)MU%o)AjDBx3ujl7((C}@S4Sz> zP^0C+yEJeH9py+Fo-4N6hN@W<43y29UA-PUQeZxV-^W`ebW*`b*Bui@04WVXmEnc- zP6P8_$_mJBKg?-L7Aef3*`i0Ty z|11AmXxq%l1JFld1o^5u!T?ynb^z|EZ$no~_itEnmK6b6;=tV3%W((rx=3V!l5TKW zR#}ZKMS-h%EyyKRYYwCNY={5HzX6@|GX_*FB=qlkLy`Oh6tD@0F{J1%k|gRT`V}23 zJw<2HRRDU+j;bNPlM&9_ijJQX-u_EVUNJ=Vo0Jv;W~};Kv^+29(=zf^)KK&Tt7r$K zTs5U8F(sdT;43(D6dB;mS<4w(V>Nw7tv+jcvbZnWdDj(~6*>A9J-P1~vKoMMPth(i z%f6!f&Eqw#+RUD+zr45FO6@fBi|(T5uFo9fvS01CO8g6GG;I5d_M+pi-*kCkbe21^ z#1w7v&{yyfsPa&B6n(lDP)YQut61;Wq(3FhytaP{Bdd6$a=FQ(aKqfF%|{$)>WI zS#d58h#=Thrj|fr=z&zFBj!*^rNf^CVCxuyk^-P`gpn+*Q7iT>*0B9HiNFqAth=VD zqH{ob6mck&^r#CEVWYwe^71J#XlViTr&H8rCH8<{6^s}_gQ|||{}ZIBcf{7|sR3%; z)%7-IYO5N3R*yFw0Q9YHis}I)M5}5*OPH4PhQ5=8%p9nBMBZ`UE%jCh1|E8G8*%Ep zAo7jNSI6o!5P1e9{=WgiDKft(N|LGF>W7ICdX<&Q0sua>xEP{F2rOBfGR{s3D!oVm z9RORa(9aI&PJIpZH^gFAEp`@*7XYo-uN#OMC|SC2kEKXdU|;jEH{`-nwBl+oybb_$yo=jH(*=5hcM>KItf8sm7> zY!-~5MSa&|bKQ1|Qy`(xaW2d8<|amaZWc}gm`#^2ox__YT32&YeGKkuABX|!HWd1% z5Oc*-0qceBp0>Yq_(R_xsN!WImraVi7SK=cl|=-q3$5E3paz??R0wT@iZusZL1bf2 z&7pG275Ur1isl1+4uIcen#)nPyy0W`^jlY;J(j>u7!t+}Nvy$ICY&_CLG zatBfh;ke;u5|B*}pmkoJru4zTqt{6(K}H=Qqe7G~V}L?YM$=hFC8|+wlT~0_zJx`n zHDy#TGU~@zkRmX;v-sse(6_J+^}29Gnkz+b3pEGcC~DqhlG@G5q=Ks0lmYa2MR&e9 z$S-u)m}{2%2|%Pi9dAPwWePOopH|3|IPnIryVpH!o1Tt`J%g)`O;6`Te|rh=gYKcy zf#d7l$5$us2O{rWxPGDBH2P7zH2TfbH?Nh#OHb@J0B$^_qkpUY(0co!dxeem6RVE5 zgZKT>HU4JiMrL#0;Z6UMPtC^9-#dD5urxAT3eVBD6&LC0TRr*q@;VedQLm+}T^+HB|17J@LDmf#Q=OY1@DA4mZ8()YjSf-MKQNXh}rVSZ>RqM7p;JW6mti?$U< z(Y^wv9GE(WDMNOI(5vV!7>#71mZm{lj#UkVs;Rf^+`0CG*JP>sTXa(Q7dXRj*_G*s zwza0MUO?o|o%0tg)Yqs8KQ)}Vc*n9E9am@_s@5=Lf8jC>kY!KNGeve2)iSB}5Sd^0 zf{C^xzRNq&V7>yzzicadf5d#nOuC9*Fi+h@H?)J!A@13kE|hqGSz}_Vmf5-!a?JZo z{$YiXHq6Gd`#bQ0W!d{fZ^onlE;5Vcn*BRuk$L;aG5=)YGHS)waTC2h$i7~)jUdT^ z1NyqYnO?8LXb=-vaW0dUL6uO~6I20x#(*drHL>F3;IVluR|^-E!jgsk0g7&Z!Nfj8^AVuz8Kj!RfC|+1Bv^TH zYosR4vjMkFN&0#u0xC>e(lKaL_fb)+g`tgZZy8TVsBqQ1r>><0>T2qe-GM$FNPS5V zaxtGSY8SBQD;P)^q#@86t*-TJR)N=Cy7{Kr0c6)a=5RFzo(4dRs({s@S!WU;1*y=n zvv|d_O=zWf;Ha*UN{|G?C4c9wACz_-`f#5m&kW%Q5-j}A$X}1#XJ7hYV1pei1-bze zyZQYa-`|QJUXLCw9hoRif2R~Zyb(=YAAi`;xYf}6aYHZE*^0*2qp_{%(e>!j4~}d^ zPp*zX42D+iEL+c$w z_n7sLgC9o@0)V!5`hK|eX6Qz!+;ZZhgCFg#5N4d&Q{K~moB7*Q?;pP1d;8?p;IZ|= zW2ODaKVm9&COXDEvD+J)K>RmGZ~AWdZXNt-LpdB-d*g=ZXZHW9G(uNjRJ>JWOUsC&jJy>5~v29Z|(y#9O#aV7Zz6g&mC zdXoS*B-N#yQ0UotVa)PBKywARCtO~Hx^$G)HaQtN|2;v5|CsO za_K$5ab1%m@J7+M92WG$p86Bus5ZaT^QeP_qov-Ho1s%HW2=!3$En{%JIdkaa`TZV zPJ4?Ban;Af?sPr!kb{S4!K+Yk(6w@5?c299>(L>zw397|8_UhpP}=E27?l=OeB+Gs z=)>mLiW4KCr<>a z`_Cwie#=Jo#>z)W9(h@3`@^P|wI8fE?XEZ>0bPb#*VJ2MHpov^nIORu-11tvsZ`~Y$!T3;$jp@Qy0hunrD>-}&V+cdz z1r>sSAoi=pD4M|?=VVb(1_9qyg}JOO7SQq+xYWGWdUupqR3~Yd3X1;=&%_1C3eG53v@Y=2JJ>VdKkk$8qgRJ!)PGR+S3EA-i6{~+5B2&Ig__!xjbChLz;HLL3j zm?PDkCTn_@Ogy&^t1vYUJAl(o!GvebyU`qUDHww_Xa`nD48m+?bqMQ$?jC4)IzTIb z3A8o1F$&(L!8)4Di&g@tQ5BpUh3&@?_tPQuVqYRJ%y<=KPRW#1J{zO4mkR%3Q zz`ueT6#UjB#M`oV?Y^gZZT7yWaqa6Bx1-Bdal9}_R4PqXFqM~y0v_F`dW)9{+%)}MAqDphRjj(Ngy;Pt9f~e?NgA1k_j)Y8&4{BLZ(}bGZ=nTn=NW0CV z4I4h4hFf=aDGoM}qN5TBNYwP7)L*dBCnj2l)ijO^gqO~Oq{L_7k+$57?K*2J7!RTG zjR^H@sf7bq#7jKB!c9>FN(%t2l2K58!YeC^h-jf2lP)=*pV7e-1Y;=FqWPgaF*Qa( zVa+iyIetOEQlzyosV0XhDFbUYxv^l$$IvhM+T z4$KjzdBwd&BI_h_tN*>hcLz(uXMao1L4nQxN83e)*c*I#AYI#==Dr&p>%^ISX4N-hqnV3Kr%gmAzH0 z-D>SwqdjYT>yU1Z)DJZDHfUuVdwoc=MVbbhdYf39J<>eT(%Ztqj!5f3TW_0Ic2(~x zlyyc{53K23GvM#_^PGWugcFKi{h7 z^s^fM+_WsE1}U3uoQ)G&m!;ReXztx2)Z-hsvM*>r>y3Krz1zZE?{>k5+#O<*P>YtE z#GPWvn1#Ki8FhCFEy&$H%KO`XP2bz&=M}3EiixrCfQYc^(NOFxJx4;)AfBExVk{UP z5(h)!U|%Q_ITh+Vhh*<@By}C?)>00J28DNs;Mj@_i6LO9@<=ScN-A>26l?b(eCIq zOmU}qfq&83YnN7V)yZ+WZ@kX;_=^<> z7MnUHqO3T>axfEDibApw?u#k*upAzgW1+!5Q85e-DVFeHOgtk>iV*?DdO8vf#lnMU z6f+AcmO&vTNug1tC>D)`<(OiYhlfzwB8aC$!;zTZsF-9ia#}G)PxULdV2~|{U{I!c z_xUF8cXvE4i;~B|e z^t8vF?TZfYbl>+jUfXeX$AV+kqNie7etFx}aXoqG)tw8D)r+308PCf*;?8}!Den<;^0ZGdVXJK zUm>Lke{9!%7lGV8H_qw%Yl0i+`l(Ew(^s1mK)#MObIKq!v)i)C0*~~}&da9Lbm(u? zpFB5e7&Z8fV;hb|pA&ttv!X8=5qy|!Q37xqjQNDHB=*Ik(x@*M^@V(c;)st1Euiqh z9=};4XrL5{=dl)LrAPwo91s=TNH}(ujk;o@1OPz`Eg-bT#?!~f69Ybgmj>~|AabP* zXsQE$@(y^D+y^CP(_3E}n0zE{Df!0pUwMAPQkk~8E|pv;nI1`1u1Qp`ne)tz{Am38 zc*4IkS-I<7>+U5ZXRTt>pcI`BM`D0~xucqy2_W@lGZD*_8V4=%ra^}S`Y25p1!F8T z{^LeH{Z&pdecw1?8aGJhag$&k!?LE<$xWjky zERyg)7+vDJ_$>88?af(n zATU;OfGr*BHQul7137RVgn}T-eSVu_VxZSM8Wyp1F;cXAgZO9wU|BInfYdFK=$SJF zw@u-}(@}}AA6wrNEpPz;A&?Y;-=bJgg=CTTg#-k`DYi`46f3pK-eVk)&nQlHL_{GN z9gZnRWXiPleX7&;)7h{f2IVM#9rC5U$lroL`P=ZYpIn}Wa{t_#o2G^G{Ylq>$%D6@ z6|cxM+g_fSj>g*}ar?kz_j?w1+Ukgxekoyq$9N#$lm7~+11(O%mrw_*4b}qPf zqo&6@c{J^+yu+EyZBxeg9HrAO-`e?}r*6?*Kht;Z+|_dn?)J2^D&=fUI2&h;?>bwT zY$&>9=bTk>QzhG@edt!k={xZ5B9gL{&f}bD6iouZ3#pZ$>iA^w_{oMIaYoe#<}aY#O?SkHvZ9~pm^`QVTi^w24L4zowdn=2n!g5Mr&kqwRCI{?x4c@EBPu;Sz@42uHQZ_u6hp|+gO|Kk`#(dK7pl?)+ z1$-xu?>i23hh!nzC&whz$~4;}(Z_}V0b5)#1*3yOsU7jL(zd78w+*as6M}6A*S8&8 z-*)2UuL&N2>$K1}p>G6vKm!J&=k7bvcLvh}L+2ujVdRv*N;-nlQYAdli1^OvfMTI1 zcpT|4W$mXd4IC0olMYZSAxjl)5_(XKgV7P`D21%(_i5td6(d55X?Qp+C}#3EDaNzn zc}A8>H2sQs;2hAaVj{j?DH;m*or{P-zAK%k+D-?h9 zWa`mp5|2I;KOT(NJ&T&hsOGK$P=P*DW|#e^BMS=Dg~BwPeb4u{iCi zyt4c9?v%%$@c85HyOW+hQ`XxhWs9ZdubjNnd%5@Jr&HdxgtskOx@uOM=G$*{-4K2<@Z*6S!CPIoPTe{g-|$$X_A$KuC^fz7@~)I;Rl>7s!Lugq z_Ac2tmuJbz*@`bYFE~@ynuN7xcGG-o+*-3>-MUy@I^B9@-Q{(&4T-Xj`SN7(mL-n2 zHQi`hbe3H@a^c8~X(qJb^nGMR;zyM8A$j*@f`@;#uJVA>_%o;ZfY-8&t0J&Mi{kD9 z%+D1L^fC_snBlt$$uk&VlmYCht2uxyaPa-Kpz|ESSPr$0G8tf&E!YnblJDNo5+)2; z8odx25{!ib6W(ipgolt<0T~5S%k$p|P;2O?{gvnRo(SgeQELUz$`AQ(^Wblap2p6f zFin_a88oBUohTYFI>S$Z1L@BonCxrD=^IEumFLhOUtSWT;Bjcy;J}p>^%p=u@+(Ns zARVV>J>qzQvhw`@O=M_hZzxYb5TW<>7VZrq~ZW=egp{;lx z^3@lN{RC+9oIcM$SJoCTwo%V+<)pI{R>6t33y?tAHN~;bdf9QS-s65^Ir1FJmW*4m zs*8YNx9gex#OLQZy)RgYyY;MP9P`6^s$NnmM*lvir)HlL|5YGi!8&1m-g@!7>T`r2 z;Z%g2_nplX)*RSDxF^pEF4q6d8YBd-wmE7o(Dnmo<^|IWzthZ&aR2vuJfihHI~p31 z;vQT57^s9i0%aLlqA|H+9uyp0PH*_eEWYsI4!h4c<~XHF63;-Ac+ziDJVIDz+=V#! zT(Iw~*mn*h%TnS^#G$YZIkn28NXIeiN(oq=Fi4L9Fv)nsbwf`N=nP0T!QK={Psu?@ z82Z4*M5Ufz6TFAe!9*+$1a=alV-M{=-zN@H6TacWzR>WQvoYVPQ6FQqd{N1VK7^&{ zAQNtd@wH&Pd}D@yeax`KKGwEhl2B&kY&atN`XW&XXV0hulC8DF=Wms06-%^$Rr-Zd zDzvv0`#|WNsESS%BL+Y*QG!a(^iX;Bk!7T6SpK2XA*yaYfloOEnSo-%SfdcS%cv)% zgy;hB<@hmG;*S17HKg3 zHjT{E=UOeo<^zI^!RH_wl%yaDIUEp#_nix}_bWE)wAx;gI#LQR$~L@FCxSIGq7kI+ zm_w(=5E(`pxd-?Rlvoj0RWs#WGOV>7;TN5rOGhspjn@X|Vn3R=J~0=)@#Ib6uOn|o zZax#QKDpq0JY8OO<@n{}sq#RgJdiBkFtz`KGVhf&m)E4r>Su&&gI5RR_3PvAz>?A8 zDY@OcCf&U1^}uU^banl;`m6Qn#@1Bh)$o_%+9H!X{T40mdv&OC~!S+%bYs+bmHLCkZdnFzj(W% zF;%hpor=}z>gH5+XQH}uzAah3ZMrD!sh-}GF0cNeu652&^=@<~+IKC~?Vh%!-E}E< zOTyhUduYKONV}_3?o|o*s#$sNz|FSQ-Y+Efej)B&wcvi@HhSr4f5+3li0?mq`S46E z)wm_mxMiVy>ynqN^x@6Piq-epx#Ienr{^~PXxH^!H!5$gU2yg+8H~k8__W(|#dg{D z9mmJ_Dmi!c9j?e$k*+#&Yv14WyxkK&@_X^$duAcn7Z=22@N_B|O$4LyXNKZKay&L1 zm!FGQj7&YUSkscK2_$L)$(jvQM?P@6rzJ$0X@t^X+x zU|qeqf6IY3?q}uhgVo&6+H40aj6Vx>H5@E8-g24`S-4xJW+dIJFjBbMcF1JB)m(M( zVdJfb%?Q73sXpv8zTMK~Y z5)n5NG0Ktyp5h7yLxbSypc#q|%E6#SLKBGu5>OM7OcMMJbW04Ns5}(ov5rtFQo=GG zN+LQ;%0eX5sc1C9V1z^zAlur~V-z|^-eq{#IaS5cLJ9kHkD8Ug3GeINZFhCjRGoH} zeXS>5<)3k+s{Dy6|Bcp#s)r|!OqVB34}*J{d}QiG(o~79HhFNWFKMbud#fi8P03WV zx*lm~lcxH#w^B{3q}+p30_FNza*Qo&O1CLk|ysRQ?PdI~s{VOC$RLKx1IT!+ns8pMs>=7ebOa6zEZ0 zgQ~t6IAb6fJ0DX#dcqJuCgjRzV*?STOiv7*k4d7)3YO?;z&-;*G3X#6>yAmIV4Mj; zi;>WfOt@JUX6n&WO_3hMI*kPTqR>@(mL;hQZ>360>_giTXy$?WwVZM-M}4i9T3Dg| z9GlQA2{#V846SsXD1{s`b>3T83`$Xx`Y|3MmnTZZ8%lI=?k+;HwinVd= zT=%@~dQY-(=agx|TDREJF>jndme{a2+0soZzQyLioIJlP(eb%tbJvvVU2CJND6{F~ ze`=Yo%!GF7Gw<3la1i(M{h0wos^%d6;&k!XgmK*X6?cv%EhCA?;Ae()g?3=;lWE5^ zZu*J^oC_Ds3^RDI0-QHW5KY1yN|3l9&!I#{KZ+6sxEPcmih?HEm?J3;GVPmoSn`W|_ds@SuOUz()^r<$NHaOpKlD*eR5m;R8_ia&zlLU>dM9X%z8q;e!Q){{tYuMUsOSi61weC!`V(q?bZDQ-v z@6AJfiX{lp)dy_Hb`kAJ>0gkS*-zPlm;Qh<7#h1qp;_{1wAs+Iv6QY;lvaoIr{sN? zyjJpB$eYo<*Ab8}!uvY+x2CnXYa1tzPXm25LTCp)){G}|zOQ;Hpj+F&nsQ7B)cNX6mgGJR(_!l_OjS3|hFpSM+H0D^RXpY!E76 z%v#7bB3_jf_W}D>OQZPMu{Sb!m+eJxO2l=lh?j7qDuLHk&`iwiBX(L6Vb&vvLIBLN z&v#Iafc5g72Di!xI$v}!G71U35Izl~3up^6Rzto}4D$L@!_Zx`cFv-v>^lc#p2nuK z_Hz3YmCl3)!((_=k2HraGxdyWl3t<&30@!J3f<9;BDj6`KD=COmp8N$q+|gWpfY(bE8%+@a2=32i#2j_ z!7}As4hGEj8|3|%ynhD|+^AeZ-lA6fEgcpw#u%U*hd1n?q1!>3O^kvjQw;^fouFZLx*3JK@u0k{uN z3$L7;-SoqU=lA_-&&_>*d5k3UB7eKN<#qpS{wdQX`vv=gwJ~jlsJ}X4t)5x`u64~v z7Am8*dtWAu`BG$bJq^bGNO%Uo~;o2j4ROPGwCJ5`A&x*kb!cAiMAAey5606YGTBh|S(l^ib6SJD<&~tlU5~43;Yy;RbC_o0D z_~a&hMi=ttZO&7Aq|?s&$!GY(H(XPL525=t!t{WOwnW2X`LT|xhw8DnE@jTQv{pIDQo!i+(om6Syf zSZOYFG0%NA!0tD|%8h-y-^vz{VrFt$b=eR=&si~y3?&G%42ILdF+juCPCgVLL+)5v z28w5d!2ccT^5gI_ONO`Y1O#8@e?9zKc>d&UIJs&^!r7t$@BsqaJ0A?QcM{0nsX=UC zm(Tc?&)nr-u4|=*%_der;0p9e7(j&bT_Kzf^9TSoaYB&I*s!E3Bc#v>NG76K0=_T6&_VD)EbTiM9t=Mb-K&&7*7Zd2vHeeUJ+?3S z$kF3_yN>o?gR+_UBfLu~>C%ZNCfir6N5oMUy3Vt0oNJM4$EzURpjVA~vU8A0yoUzm zP4GyvY_wHn^z5w(PiwqwThg;V?%e+IqPyXT*0{?Lv7)W&Jx?Wsk+v#Tq5ajGdhYwK z4&%S!JB-T|+O*!;Ad~)VCHt!Lyo|#v82-%eKw-_D&`_HphroCnc{$%FjUzth&U~1r zLHI{1MOI3a9KnYdF1+xLYr||SNw(wG4GeSnoATH(>EEKfy66dxsjL_yZ)z#ZXq@j* zC`0H-45ns8mD0x^js1p@NN@V?G{Xfl`^maH5^h}~0=PL`+3y^{x~_CuJM zuB@9pJZ(yv>e3apkVM9krrI1yWLX8ufz7K(SFV}r^M z$8z(l7b?3UtK6P6buXDrX8)}5b<1m(B@V%P4}08v@-D@eoCfpSxixn=JeI5$bM5T@ zyBr=%WyqX4aqWq#Am9+3>sD)7wOU+J$%h8Bx$3SFp(P3}m9RA4+{w~DG$5o>B5XV^ zU>KGyLPD=>5z_qOWERi#{v7O(At;_OKth7_d{=Pa=NR>oWN4DV*t`P4V*DLiHDyf& zPv_`jFFp$EWnM7T8iEaT(b(qw5MKh-GWH2aVq!j~it=f465nY^GFhIclY-(B6tgsh z`VukwJ<_+}|N5Wc{6=+=5-ed_H2@bHHyFK>c1xDNO&L^aIROB*n5cFlh=QcvlHNff z9^glZ;7Oz>Gl_(GH>Cb4<6g{ z6>2`Pw`58-ZY*TJrE{)8CC4}}V{Eos@87sNwnh*4x9g$oQy1n)9vo&_pH0_8diyZd zVk*0sU(YEdGK5)_Mwa3*%^`u)Yh64JL}EMv2*oofHNyETdK6pV@PKBaBBND^>=9=& zd|jD#5=JOR^11NPU`X+1b92=}<=Nz{Vn}gkQ&oAtY8JDsam7oPGHTMY7PHB@EoM`- z7TuI0=c?-1TbHFUW1Kt%9(8Sbl8cgbeGzH&*hpvzs!~I7(A#tkp@S-D5k`^us}#c! zWQ}xo@PI^yz8;LIM8u~`_c3B}8%3|uP#fjpfjkMnV#FI%34fk&pXpr8;g~oeO9XaR zgJTO$F~Zoo4@Ys3qnN|;U}(@^#E>{c-ikpu&0ek;&xxao@i`m{BKCxMd0iz}or3o8wa5g}Pfue_8R=jnP*?7y1*GepqF6T<>_da>qLrPbF== zlY3#@aEB`^E?O+}PDheuEwiC_%2qEKk#f7HE>*KGQL`>pvmsHl;f5(*vtglT_mUZT z;PD%qQVp9D4VzL8+Y$}iZkpl^+ZG!3FIgzp$~E{>^&1oQ8&maL67^ed2=V$Y3-#Sg zHp;bgXt}yQQQe-Z4kW4r^Fq8juu#2o$w9eJuC~75dy6TzWTp4IDA&ExdrK*|Y^C>l zD7PHB*BY-jrmEK^s@Ki$i&w8(sNTNhrM!y5d6kq`#roTxsA*SCM({N?frXl#OVyNH zvr=v?<<@c4wW+FgiK=y}stt)M8sn-B3st+9>M6H@)upjd)oe=CY`U>8UbAVTrfaE@ za($e)lFVbS^jz+lYn|>{DDPNmqKxJO87;_|+`qJn^VVEBa{0*1$LOq7!v*Vym(zXlRrn|qj8)goa_RYg|c&dS@y3)y`zqzxStL){O z+~Ac}haJ`feHmsN>k)L4O2RuCbvJI9bY-QAS&E85V8euQ@JLp=n9UXVbAZNDH8@SQ zlJBwy9CfIxLsRgkFr^llR=p4%s2W$0$?&I?QQ@~ke``U-E{J`0cB4qg3s5T*#WGTz z!m3^v`f7YLyL@4pieTI%8^+Dj&xqdr;x|F(!pOJ~y@(S|1gw}7s7wL+4(6`bLZiCe zoc!XZ!fz-@hv0Gxz3&s&aWk;iiE%5KsxM@XpbOD7IcB+ydatujuuS$9zrgW*BaLu& zW-nd@ZZt`obLewkaI#q_*89e00bGLWR+k?Q%jeJaCsk?F{Ka1s zmg6I*M9tCYG~0+yu_2VNk{i`bzWtTz=_Vb`7sQ`vmtb$@EvPl&~DkUtOkw9iUK32r^uz{?&@4j2D z!@3%v7eZA18*00T$sQy|SM5XslY}w3hGIH179NrWN)*XEP2L=Nv|?3_uW=|kB-s#E zY$Q2RZ=f)dK3g%-Yt)+&80q#8REfy}e@-Eykfi@c9!Zp#FwkGDl3ujrvtI*qUos+Q zqhwWsLuaIZDnkMm#Sx|}R&cBgmtUxza^!rh^gpQJ5)2*QPtSO*y9@ zOIWK?R;FAzl&}WUn?JWa@mRvT@dKQPs=iP?y)|CHA>rA8lV7&2khYuKZgehMOD;Js zIHr%zHZ53NA*wgGeOSa3rGf-#q{J^YNZ%;?JCl@9Ilz?0eTOkd3X#e^|mf zy^E#Z=`(R(XQHBW{-I4Wp-5fvPw6OiyyY3#+M;+(y6~W?p(e0T& znshhKb|u_xxX@$X`OC`PbEX;7HOH%tcPi;L+G1tR4d*XwpPV<{GX0I~7p|Fo*N(q> z{GHk-la)`U-7pSqyyqzL6iptvSHgMyb9Kq`EvfQdiSk{^@;#8Mo7+`F3fhk;$Mor$ z6Epj#d)~3OK+tY(M<1+?sa;7+)lBC*mb!H7hWWBY>!wue!->|1QT%Aa+LVTUoFUQj zP^x8jqGdM{k3dM6>|}9wfcf zafrYV^K?nj*p8F9+(h<`Mu&aj)AW#hbWaf7qtG`j(Me92p1=--o$UbRW2XLOiy{W- z6oyeA#Ys;nOPPs&zKs3r=#fMo-7h2)V~1M@93^yE@ufYp<#9(_+|;JZBaxSJ8coLC z9Yj*rESDKA*##p^a$uM%m~u>TiV!#F#9^u{6y?NWtSeY@;xN}0tT}PJV1r4m1BN^f zIs-BE7`wV*?5t*q!|u$XOCiWghZ)-FCv4TzyU@weO%J~4X=LV{<<-vT<{IhXl&xy& z*-Z; zo}V;tSgD*!62U@MdJ^>&gLDdNXWVzkso0(2Taj{1v?Q-0vZQxGg-hY@WX#+DyFRY8Ci98$mC6++ z^gq2cOz+1^awu#N?~<36`&~?g-y*pv%vSwcB;=5|bp4a`Pn1@P(6YsE&$c&Ko&{9d z)bmv1UDRb8Amyk{IBIbcKMjJ&dD$6n>`1sfQm(BD*VYBs_C>FDK0of?o~YS=V^h-m zIqZzmEjPOsSFeBl%ddTTzVipqPw!hOCmOJH3nV=5@|3GF;cARq8}FI9ikd4=U4CjN zc5VFXcs$URXzaR)HV(2jZWV3=+bh$;R4fyOxlexk_p$9tx8#<;FIVBq-ZIv9yNy5f z)^@Kp{)OAz-D>%Z)#mPZ94$f&^ff<=vO6@xowllqHH~5 z-Yo6KD70WZ2^!cfEljR)pARq-?`1Pfc#x>^yJR*)GpinsB8ET{iF_9sP+{@sbBu-V zTX#i51E+-0p0U;0C7XS}9M~0!!Zt_VgMeD_GQM86qpeA9u5<45*EdZ<0>zqE8FVJ+ zuuG*V#sD*f50U68)%1th8JLz2MZz&=X2IkQ3~1wccT|dLCUiI>oRPc8IPSu1HqISoPhmy4q-{cdudy|#j)26h$Hsx+kxSMD9z1jAo z_UrBQk0;xA##?s1>xN1P4ML?;j4RUW63)8U8sg5n1?Prz$Btk6cFr};md+12n1br3ZO(49Y1vep@JWpH{m%%jxQTnRq;Da@7fUh2C!iV4 z=}Q|#@B@gQ+$$6dZ9T#GX5MD~EE@r4J*zaMzz}YwgmDkuGD++gyKKlud`JwcJ|Q4A zVs^2djk1UfRdIbKZZo5cc#EP^7#uF!AfOL$<+FNfR#iV%QS`syW0ew;LInru=sRvz z5*14D-7E8xu;7+R;>3aYlgt#asyD^=<}jq zH~P#q4@IttOhT!ao)788md`Z*+7TNv;#4J}ik%jT%C+M*ilX5mx*<>Mz;r03xa?8H zg^~wwHM79XLK&9UrkYO5u+>8#RM!PN$XFbDX~;oGbHPha@*mJl53rfOUDhybikG%c z72&MBtyDMSYE>1k3(oaqNSd2WdQHd~0UX>FQ^gXIMNLu<^NGP?W zw9}n-d8dk(%t*g4`|!r!uYH*N!`iMA?kBb`hw&{tPkxE5tJe5d4G*7W&p5sd+u9z# zv!GF@2c4G)en@Of)KZiv7m4608#QS&g$QY|^`=T#Nf9oQM*vA4M&^s$hX$kBde>fL zZh@khtP`Qe)T|Rpq3L94Rk^t4=)r|`!B3qXo9;U%-!QWjU zUUP)wR~&Fa8^Tt3E=x2L@%9xD9TCb}9cJzH86MG?e6l+NiFr~bv0Yp;qQbMv1?8Fw44UEsnm+U*U1#?iWljc>g0K?gq4s{qbWWq_1-L zIT6y=)6fk0@~_`z2CM7
-

UI Assistant

+

BrowserGym

@@ -151,6 +151,15 @@

UI Assistant

var USER_MESSAGE_RECEIVED = false; + function escapeHtml(unsafe) { + return unsafe + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + } + function addChatMessage(role, msg) { const chatBody = document.getElementById('chatBody'); const chatDebug = document.getElementById('chatDebug'); @@ -158,7 +167,7 @@

UI Assistant

msgContainer.className = 'message'; const text = document.createElement('p'); - text.innerHTML = msg; + text.innerHTML = escapeHtml(msg); const assistant_img = document.createElement('img'); assistant_img.src = assistant_image_data; diff --git a/core/src/browsergym/core/chat_files/chatbox_modern.html b/core/src/browsergym/core/chat_files/chatbox_modern.html new file mode 100644 index 00000000..63b9d4ac --- /dev/null +++ b/core/src/browsergym/core/chat_files/chatbox_modern.html @@ -0,0 +1,354 @@ + + + + + + + UI Assistant Chat + + + + +
+
+
+
+ +
+
+
+
+
+
+ + + +
+
+
+
+ + + + + + \ No newline at end of file diff --git a/core/src/browsergym/core/chat_files/img/send.svg b/core/src/browsergym/core/chat_files/img/send.svg new file mode 100644 index 00000000..7d5705f5 --- /dev/null +++ b/core/src/browsergym/core/chat_files/img/send.svg @@ -0,0 +1,3 @@ + + + diff --git a/core/src/browsergym/core/constants.py b/core/src/browsergym/core/constants.py index e1e59a7b..3169920d 100644 --- a/core/src/browsergym/core/constants.py +++ b/core/src/browsergym/core/constants.py @@ -1,4 +1,7 @@ TEXT_MAX_LENGTH = 2**32 - 1 BROWSERGYM_ID_ATTRIBUTE = "bid" # Playwright's default is "data-testid" +BROWSERGYM_VISIBILITY_ATTRIBUTE = "browsergym_visibility_ratio" +BROWSERGYM_SETOFMARKS_ATTRIBUTE = "browsergym_set_of_marks" + EXTRACT_OBS_MAX_TRIES = 5 diff --git a/core/src/browsergym/core/env.py b/core/src/browsergym/core/env.py index 1476536a..0898e23b 100644 --- a/core/src/browsergym/core/env.py +++ b/core/src/browsergym/core/env.py @@ -4,22 +4,25 @@ import numpy as np import playwright.sync_api import time +import re from abc import ABC from pathlib import Path -from typing import Optional, Literal +from typing import Optional from .chat import Chat from .task import AbstractBrowserTask -from .spaces import Unicode, AnyDict +from .spaces import Unicode, AnyDict, AnyBox from .constants import TEXT_MAX_LENGTH, BROWSERGYM_ID_ATTRIBUTE, EXTRACT_OBS_MAX_TRIES from .observation import ( _pre_extract, _post_extract, extract_screenshot, extract_dom_snapshot, + extract_dom_extra_properties, extract_merged_axtree, extract_focused_element_bid, + MarkingError, ) from .action.base import execute_python_code from .action.highlevel import HighLevelActionSet @@ -28,45 +31,65 @@ class BrowserEnv(gym.Env, ABC): + """The main BrowserGym class, which encapsulates instruction-following Web browsing into a Gymnasium environment.""" + # gym metadata metadata = {"render_modes": None} def __init__( self, + # task-related arguments task_entrypoint: type[AbstractBrowserTask], + task_kwargs: dict = {}, + viewport: Optional[dict] = None, # will override the task's viewport + slow_mo: Optional[int] = None, # will override the task's slow_mo + timeout: Optional[int] = None, # will override the task's timeout + # interactive / debugging arguments headless: bool = True, - viewport: dict = {"width": 1280, "height": 720}, - slow_mo: int = 1000, # in milliseconds - timeout: int = 5000, wait_for_user_message: bool = False, - demo_mode: Literal["off", "default", "only_visible_elements"] = "off", - record_video_dir: str = None, - playwright_kwargs: dict = {}, + resizeable_window: bool = False, + record_video_dir: Optional[str] = None, + pw_chromium_kwargs: dict = {}, + pw_context_kwargs: dict = {}, + # agent-related arguments action_mapping: Optional[callable] = HighLevelActionSet().to_python_code, - **task_kwargs, ): + """ + Instantiate a ready to use BrowserEnv gym environment. + + Args: + task_entrypoint: a callable that returns a new task object from a seed. Used for creating a new task during `reset()`. + task_kwargs: additional arguments passed to `task_entrypoint`. + viewport: desired viewport size. This will override the value defined by the task, which might change its behaviour and difficulty. Should only be set for debugging/testing. + slow_mo: desired slow_mo value for Playwright. This will override the value defined by the task, which might change its behaviour and difficulty. Should only be set for debugging/testing. + timeout: desired timeout value for Playwright. This will override the value defined by the task, which might change its behaviour and difficulty. Should only be set for debugging/testing. + headless: whether the browser should run in headless mode or not. This will affect the viewport size, which might change the behaviour and difficulty of the task. Headless mode should only be disabled for debugging/testing. + wait_for_user_message: whether the environment should pause and wait for a user message in the chat after a new message is sent by the agent. Useful for running agents in interactive mode. + resizeable_window: whether the browser window should be resizeable or not. This will affect the viewport size, which might change the behaviour and difficulty of the task. Should only be set for debugging/testing. + record_video_dir: if set, indicates a directory to which viewport videos will be recorded. + pw_chromium_kwargs: extra parameters for the playwright Browser. Should only be used for debugging/testing. + pw_context_kwargs: extra parameters for the playwright BrowserContext. Should only be used for debugging/testing. + action_mapping: if set, the environment will use this function to map every received action to executable Python code. + + """ super().__init__() self.task_entrypoint = task_entrypoint - self.task_kwargs = task_kwargs - self.headless = headless + self.task_kwargs = dict(**task_kwargs) self.viewport = viewport self.slow_mo = slow_mo self.timeout = timeout + self.headless = headless self.wait_for_user_message = wait_for_user_message - self.demo_mode = demo_mode - self.action_mapping = action_mapping + self.resizeable_window = resizeable_window self.record_video_dir = record_video_dir + self.pw_chromium_kwargs = pw_chromium_kwargs + self.pw_context_kwargs = pw_context_kwargs + self.action_mapping = action_mapping # task self.task = None # playwright - self.playwright_kwargs = playwright_kwargs - self.playwright_kwargs.setdefault("headless", self.headless) - self.playwright_kwargs.setdefault("slow_mo", self.slow_mo) - self.playwright_kwargs.setdefault( - "args", [f"--window-size={self.viewport['width']},{self.viewport['height']}"] - ) self.browser: playwright.sync_api.Browser = None self.context: playwright.sync_api.BrowserContext = None self.page: playwright.sync_api.Page = None @@ -93,14 +116,15 @@ def __init__( ), "active_page_index": gym.spaces.Box(low=0, high=255, dtype=int), "url": Unicode(min_length=0, max_length=TEXT_MAX_LENGTH), - "screenshot": gym.spaces.Box( - 0, - 255, - shape=(viewport["height"], viewport["width"], 3), + "screenshot": AnyBox( + low=0, + high=255, + shape=(-1, -1, 3), dtype=np.uint8, - ), # swapped axes (height first) + ), # swapped axes (height, width, RGB) "dom_object": AnyDict(), "axtree_object": AnyDict(), + "extra_element_properties": AnyDict(), "focused_element_bid": Unicode(min_length=0, max_length=TEXT_MAX_LENGTH), "last_action": Unicode(min_length=0, max_length=TEXT_MAX_LENGTH), "last_action_error": Unicode(min_length=0, max_length=TEXT_MAX_LENGTH), @@ -124,39 +148,67 @@ def close(self): self.task = None def reset(self, seed=None, *args, **kwargs): - # we need the following line to seed self.np_random super().reset(seed=seed, *args, **kwargs) + self.np_random = None # make sure all randomness is handled by the task if self.task: self.task.teardown() self.context.close() self.chat.close() - else: - pw: playwright.sync_api.Playwright = _get_global_playwright() - # important: change playwright's test id attribute from "data-testid" to "bid" - pw.selectors.set_test_id_attribute(BROWSERGYM_ID_ATTRIBUTE) - self.browser = pw.chromium.launch(**self.playwright_kwargs) + self.browser.close() + + # create a new task + self.task = self.task_entrypoint(seed=seed, **self.task_kwargs) + + def override_property(task, env, property): + """Extract property value from env if not None, otherwise from task.""" + env_value = getattr(env, property) + task_value = getattr(task, property) + if env_value is None: + return task_value + else: + logging.warning( + f"Overriding the task's {property} parameter ({repr(task_value)} => {repr(env_value)}). This might change the task's behaviour and difficulty." + ) + return env_value + + # fetch task's desired parameters for browser setup + viewport = override_property(self.task, self, "viewport") + slow_mo = override_property(self.task, self, "slow_mo") + timeout = override_property(self.task, self, "timeout") + + # use the global Playwright instance + pw: playwright.sync_api.Playwright = _get_global_playwright() + # important: change playwright's test id attribute from "data-testid" to "bid" + pw.selectors.set_test_id_attribute(BROWSERGYM_ID_ATTRIBUTE) + + # create a new browser + self.browser = pw.chromium.launch( + headless=self.headless, + slow_mo=slow_mo, + args=( + [f"--window-size={viewport['width']},{viewport['height']}"] + if self.resizeable_window + else None + ), + # will raise an Exception if above args are overriden + **self.pw_chromium_kwargs, + ) # create a new browser context for pages - t_before = time.time() self.context = self.browser.new_context( - no_viewport=True, + no_viewport=True if self.resizeable_window else None, + viewport=viewport, record_video_dir=( Path(self.record_video_dir) / "task_video" if self.record_video_dir else None ), - record_video_size=self.viewport, - ) - # create the chat at the same time to make sure videos are synced - self.chat = Chat( - headless=self.playwright_kwargs["headless"], - chat_size=(500, max(self.viewport["height"], 800)), - record_video_dir=self.record_video_dir, + record_video_size=viewport, + # will raise an Exception if above args are overriden + **self.pw_context_kwargs, ) - t_after = time.time() - recording_start_time = (t_before + t_after) / 2 # recording start time # set default timeout - self.context.set_default_timeout(self.timeout) + self.context.set_default_timeout(timeout) # hack: keep track of the active page with a javascript callback # there is no concept of active page in playwright @@ -188,13 +240,19 @@ def reset(self, seed=None, *args, **kwargs): """ ) + # create the chat + self.chat = Chat( + headless=self.headless, + chat_size=(500, max(viewport["height"], 800)), + record_video_dir=self.record_video_dir, + ) + # create a new page self.page = self.context.new_page() + recording_start_time = time.time() - # create and setup a new task - task_seed = self.np_random.integers(np.iinfo(np.int32).max + 1) - self.task = self.task_entrypoint(**self.task_kwargs) - goal, info = self.task.setup(seed=task_seed, page=self.page) + # setup the task + goal, task_info = self.task.setup(page=self.page) # initialize the chat self.chat.add_message( @@ -224,14 +282,27 @@ def reset(self, seed=None, *args, **kwargs): # extract obs and info from environment obs = self._get_obs() + info = {} + info["task_info"] = task_info + + # TODO this is a bit hacky, find a better solution to record videos if self.record_video_dir: info["recording_start_time"] = recording_start_time + info["recording_file"] = str(self.page.video.path()) + info["chat"] = { + "recording_start_time": self.chat.recording_start_time, + "recording_file": str(self.chat.page.video.path()), + } return obs, info def step(self, action: str) -> tuple: self.last_action = action + info = {} + info["action_exec_start"] = time.time() + info["action_exec_timeout"] = 0 + # try to execute the action try: if self.action_mapping: @@ -246,6 +317,11 @@ def step(self, action: str) -> tuple: self.last_action_error = "" except Exception as e: self.last_action_error = f"{type(e).__name__}: {e}" + match = re.match("TimeoutError: Timeout ([0-9]+)ms exceeded.", self.last_action_error) + if match: + info["action_exec_timeout"] = float(match.groups()[0]) / 1000 # ms to sec + + info["action_exec_stop"] = time.time() # wait a bit (for the JavaScript callback to set the active page) time.sleep(0.5) # wait for JS events to be fired (half a second) @@ -262,7 +338,8 @@ def step(self, action: str) -> tuple: self._wait_for_user_message() # extract reward, done, user_message, info (task-specific) - reward, done, user_message, info = self._task_validate() + reward, done, user_message, task_info = self._task_validate() + info["task_info"] = task_info # add any user message sent by the task to the chat if user_message: @@ -287,7 +364,7 @@ def _task_validate(self): # safety fix, in case validate() did mess up the active page and/or page history if prev_active_page != self.page or prev_page_history != self.page_history: - logging.warning( + logging.info( "The active page and / or page history has changed during task.validate(). A recovery fix will be applied." ) self.page = prev_active_page @@ -363,13 +440,16 @@ def _get_obs(self): dom = extract_dom_snapshot(self.page) axtree = extract_merged_axtree(self.page) focused_element_bid = extract_focused_element_bid(self.page) - except playwright.sync_api.Error as e: + extra_properties = extract_dom_extra_properties(dom) + except (playwright.sync_api.Error, MarkingError) as e: err_msg = str(e) # try to add robustness to async events (detached / deleted frames) if retries_left > 0 and ( "Frame was detached" in err_msg or "Frame with the given frameId is not found" in err_msg or "Execution context was destroyed" in err_msg + or "Frame has been detached" in err_msg + or "Cannot mark a child frame without a bid" in err_msg ): logging.warning( f"An error occured while extracting the dom and axtree. Retrying ({retries_left}/{EXTRACT_OBS_MAX_TRIES} tries left).\n{repr(e)}" @@ -402,6 +482,7 @@ def _get_obs(self): "screenshot": extract_screenshot(self.page), "dom_object": dom, "axtree_object": axtree, + "extra_element_properties": extra_properties, "focused_element_bid": focused_element_bid, "last_action": self.last_action, "last_action_error": self.last_action_error, diff --git a/core/src/browsergym/core/javascript/frame_mark_elements.js b/core/src/browsergym/core/javascript/frame_mark_elements.js index 311ecb5c..3358810d 100644 --- a/core/src/browsergym/core/javascript/frame_mark_elements.js +++ b/core/src/browsergym/core/javascript/frame_mark_elements.js @@ -2,11 +2,7 @@ * Go through all DOM elements in the frame (including shadowDOMs), give them unique browsergym * identifiers (bid), and store custom data in the aria-roledescription attribute. */ -var { innerWidth: windowWidth, innerHeight: windowHeight } = window; -var scrollX = window.scrollX || document.documentElement.scrollLeft; -var scrollY = window.scrollY || document.documentElement.scrollTop; - -([parent_bid, bid_attr_name, iframe_position, super_iframe_offset]) => { +async ([parent_bid, bid_attr_name]) => { // standard html tags // https://www.w3schools.com/tags/ @@ -25,30 +21,39 @@ var scrollY = window.scrollY || document.documentElement.scrollTop; "svg", "table", "tbody", "td", "template", "textarea", "tfoot", "th", "thead", "time", "title", "tr", "track", "tt", "u", "ul", "var", "video", "wbr" ]; - - if (super_iframe_offset == null) { - - iframe_offset = { x: scrollX, y: scrollY, right: windowWidth, bottom: windowHeight }; - } - else { - [super_x, super_y, super_right, super_bottom] = [super_iframe_offset["x"], super_iframe_offset["y"], super_iframe_offset["right"], super_iframe_offset["bottom"]]; - - x = Math.max(-iframe_position.x, 0); - y = Math.max(-iframe_position.y, 0); - right = Math.min(...[super_right, windowWidth, super_right - iframe_position.x]); - bottom = Math.min(...[super_bottom, windowHeight, super_bottom - iframe_position.y]); - iframe_offset = { x: x, y: y, right: right, bottom: bottom }; - } + const set_of_marks_tags = [ + "input", "textarea", "select", "button", "a", "iframe", "video", "li", "td", "option" + ]; let browsergym_first_visit = false; // if no yet set, set the frame (local) element counter to 0 - if (!("browsergym_frame_elem_counter" in window)) { - window.browsergym_frame_elem_counter = 0; + if (!("browsergym_elem_counter" in window)) { + window.browsergym_elem_counter = 0; + window.browsergym_frame_id_generator = new IFrameIdGenerator(); browsergym_first_visit = true; } + // mechanism for computing all element's visibility + // the intersection observer will set the visibility ratio of elements entering / exiting the viewport + // a set is used to keep track of not-yet-visited elements + let elems_to_be_visited = new Set() + let intersection_observer = new IntersectionObserver( + entries => { + entries.forEach(entry => { + let elem = entry.target; + elem.setAttribute('browsergym_visibility_ratio', Math.round(entry.intersectionRatio * 100) / 100); + if (elems_to_be_visited.has(elem)) { + elems_to_be_visited.delete(elem); + } + }) + }, + { + threshold: [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] + } + ) // get all DOM elements in the current frame (does not include elements in shadowDOMs) let elements = Array.from(document.querySelectorAll('*')); + let som_buttons = []; i = 0; while (i < elements.length) { const elem = elements[i]; @@ -64,10 +69,14 @@ var scrollY = window.scrollY || document.documentElement.scrollTop; i++; // we will mark only standard HTML tags if (!elem.tagName || !html_tags.includes(elem.tagName.toLowerCase())) { - // console.log(`Skipping element ${elem.outerHTML}`) + // Skipping element continue; // stop and move on to the next element } - // console.log(`Processing element ${elem.outerHTML}`) + // Processing element + // register intersection callback on element, and keep track of element for waiting later + elem.setAttribute('browsergym_visibility_ratio', 0); + elems_to_be_visited.add(elem); + intersection_observer.observe(elem); // write dynamic element values to the DOM if (typeof elem.value !== 'undefined') { elem.setAttribute("value", elem.value); @@ -81,7 +90,7 @@ var scrollY = window.scrollY || document.documentElement.scrollTop; elem.removeAttribute("checked"); } } - // add the element global id to a custom HTML attribute + // add the element global id (browsergym id) to a custom HTML attribute // https://playwright.dev/docs/locators#locate-by-test-id // recover the element id if it has one already, else compute a new element id let elem_global_bid; @@ -93,100 +102,169 @@ var scrollY = window.scrollY || document.documentElement.scrollTop; elem_global_bid = elem.getAttribute(bid_attr_name); } else { - let elem_local_id = window.browsergym_frame_elem_counter++; + let elem_local_id = null; + // iFrames get alphabetical ids: 'a', 'b', ..., 'z'. + // if more than 26 iFrames are present, raise an Error + if (['iframe', 'frame'].includes(elem.tagName.toLowerCase())) { + elem_local_id = `${window.browsergym_frame_id_generator.next()}`; + if (elem_local_id.length > 1) { + throw new Error(`More than 26? Such iFrames. BrowserGym not like.`); + } + } + // other elements get numerical ids: '0', '1', '2', ... + else { + elem_local_id = `${window.browsergym_elem_counter++}`; + } if (parent_bid == "") { elem_global_bid = `${elem_local_id}`; } else { - elem_global_bid = `${parent_bid}-${elem_local_id}`; + elem_global_bid = `${parent_bid}${elem_local_id}`; } elem.setAttribute(bid_attr_name, `${elem_global_bid}`); } + // Hack: store custom data inside the aria-roledescription attribute (will be available in DOM and AXTree) // - elem_global_bid: global element identifier (unique over multiple frames) // TODO: add more data if needed (x, y coordinates, bounding box, is_visible, is_clickable etc.) - - let [rect, is_in_viewport] = getElementPositionInfo(elem, iframe_offset, iframe_position); - let left = (rect.left + iframe_position.x).toString(); - let top = (rect.top + iframe_position.y ).toString(); - let right = (rect.right + iframe_position.x ).toString(); - let bottom = (rect.bottom + iframe_position.y).toString(); - let center_x = ((rect.left + rect.right) / 2 + iframe_position.x).toString(); - let center_y = ((rect.top + rect.bottom) / 2 + iframe_position.y).toString(); - - elem.setAttribute("browsergym_center", `(${center_x}, ${center_y})`); - elem.setAttribute("browsergym_bounding_box", `(${left}, ${top}, ${right}, ${bottom})`); - elem.setAttribute("browsergym_is_in_viewport", `${is_in_viewport}`); - let original_content = ""; if (elem.hasAttribute("aria-roledescription")) { original_content = elem.getAttribute("aria-roledescription"); } - let new_content = `${elem_global_bid}_${left}_${top}_${center_x}_${center_y}_${right}_${bottom}_${is_in_viewport}_${original_content}` + let new_content = `${elem_global_bid}_${original_content}` elem.setAttribute("aria-roledescription", new_content); + // set-of-marks flag (He et al. 2024) + // https://github.com/MinorJerry/WebVoyager/blob/main/utils.py + elem.setAttribute("browsergym_set_of_marks", "0"); + // click at center activates self or a child + if (["self", "child"].includes(whoCapturesCenterClick(elem))) { + // has valid tag name, or has click event, or triggers a pointer cursor + if (set_of_marks_tags.includes(elem.tagName.toLowerCase()) || (elem.onclick != null) || (window.getComputedStyle(elem).cursor == "pointer")) { + let rect = elem.getBoundingClientRect(); + let area = (rect.right - rect.left) * (rect.bottom - rect.top); + // area is large enough + if (area >= 20) { + // is not a child of a button (role, type, tag) set to be marked + if (som_buttons.every(button => !button.contains(elem))) { + // is not the sole child of span that has a role and is set to be marked + let parent = elem.parentElement; + if (!(parent && parent.tagName.toLowerCase() == "span" && parent.children.length === 1 && parent.getAttribute("role") && parent.getAttribute("browsergym_set_of_marks") === "1")) { + // all checks have passed, flag the element for inclusion in set-of-marks + elem.setAttribute("browsergym_set_of_marks", "1"); + if (elem.matches('button, a, input[type="button"], div[role="button"]')) { + som_buttons.push(elem) + } + // lastly, remove the set-of-marks flag from all parents, if any + while (parent) { + if (parent.getAttribute("browsergym_set_of_marks") === "1") { + parent.setAttribute("browsergym_set_of_marks", "0") + } + parent = parent.parentElement; + } + } + } + } + } + } + } + + warning_msgs = new Array(); + + // wait for all elements to be visited for visibility + let visibility_marking_timeout = 1000; // ms + try { + await until(() => elems_to_be_visited.size == 0, visibility_marking_timeout); + } catch { + warning_msgs.push(`Frame marking: not all elements have been visited by the intersection_observer after ${visibility_marking_timeout} ms`); } - return iframe_offset; + // disconnect intersection observer + intersection_observer.disconnect(); + return warning_msgs; } -function getElementPositionInfo(element, iframe_offset, iframe_position) { - var rect = element.getBoundingClientRect(); - let x = (rect.left + rect.right) / 2 ; - let y = (rect.top + rect.bottom) / 2 ; - //loop over element ancestors (parent) and refine iframe offset to be the most precise possible - var parent = element.parentElement; - parent_iframe_offset = { x: 0, y: 0, right: windowWidth, bottom: windowHeight }; - while (parent !== null) { - var parent_rect = parent.getBoundingClientRect(); - parent_iframe_offset["x"] = Math.max(parent_rect.left , parent_iframe_offset["x"] ); - parent_iframe_offset["y"] = Math.max(parent_rect.top , parent_iframe_offset["y"] ); - parent_iframe_offset["right"] = Math.min(parent_rect.right , parent_iframe_offset["right"] ); - parent_iframe_offset["bottom"] = Math.min(parent_rect.bottom , parent_iframe_offset["bottom"] ); - parent = parent.parentElement; - } - var is_in_viewport = ( - x >= iframe_offset["x"] && - y >= iframe_offset["y"] && - x <= iframe_offset["right"] && - y <= iframe_offset["bottom"] - ); - //this features is broken for the moment - var NotBehindParent = ( - x >= parent_iframe_offset["x"] && - y >= parent_iframe_offset["y"] && - x <= parent_iframe_offset["right"] && - y <= parent_iframe_offset["bottom"] - ); - - var isVisible = (typeof element.offsetWidth === 'undefined' || typeof element.offsetHeight === 'undefined') || (element.offsetWidth > 0 && element.offsetHeight > 0); - - // Return true if the element is both in the viewport and has non-zero dimensions - return [rect, (is_in_viewport && isVisible && IsInFront(element))? 1 : 0]; +async function until(f, timeout, interval=40) { + return new Promise((resolve, reject) => { + const start_time = Date.now(); + // immediate check + if (f()) { + resolve(); + } + // loop check + const wait = setInterval(() => { + if (f()) { + clearInterval(wait); + resolve(); + } else if (Date.now() - start_time > timeout) { + clearInterval(wait); + reject(); + } + }, interval); + }); } -function IsInFront(element){ +function whoCapturesCenterClick(element){ var rect = element.getBoundingClientRect(); var x = (rect.left + rect.right) / 2 ; var y = (rect.top + rect.bottom) / 2 ; - var newElement = elementFromPoint(x, y); //return the element in the foreground at position (x,y) - if(newElement){ - if(newElement === element) - return true; + var element_at_center = elementFromPoint(x, y); // return the element in the foreground at position (x,y) + if (!element_at_center) { + return "nobody"; + } else if (element_at_center === element) { + return "self"; + } else if (element.contains(element_at_center)) { + return "child"; + } else { + return "non-descendant"; } - return false; } function elementFromPoint(x, y) { - let node = document.elementFromPoint(x, y); + let dom = document; + let last_elem = null; + let elem = null; - let child = node?.shadowRoot?.elementFromPoint(x, y); + do { + last_elem = elem; + elem = dom.elementFromPoint(x, y); + dom = elem?.shadowRoot; + } while(dom && elem !== last_elem); - while (child && child !== node) { - node = child; - child = node?.shadowRoot?.elementFromPoint(x, y); + return elem; +} + +// https://stackoverflow.com/questions/12504042/what-is-a-method-that-can-be-used-to-increment-letters#answer-12504061 +class IFrameIdGenerator { + constructor(chars = 'abcdefghijklmnopqrstuvwxyz') { + this._chars = chars; + this._nextId = [0]; + } + + next() { + const r = []; + for (const char of this._nextId) { + r.unshift(this._chars[char]); + } + this._increment(); + return r.join(''); } - return child || node; + _increment() { + for (let i = 0; i < this._nextId.length; i++) { + const val = ++this._nextId[i]; + if (val < this._chars.length) { + return; + } + this._nextId[i] = 0; + } + this._nextId.push(0); + } + + *[Symbol.iterator]() { + while (true) { + yield this.next(); + } + } } diff --git a/core/src/browsergym/core/javascript/frame_unmark_elements.js b/core/src/browsergym/core/javascript/frame_unmark_elements.js index 4a29f15f..578a47b9 100644 --- a/core/src/browsergym/core/javascript/frame_unmark_elements.js +++ b/core/src/browsergym/core/javascript/frame_unmark_elements.js @@ -23,7 +23,7 @@ if (elem.hasAttribute("aria-roledescription")) { let content = elem.getAttribute("aria-roledescription"); // TODO: handle more data if needed - let n_data_items = 8; // bid, bbox_left, bbox_top, center_x, center_y, bbox_right, bbox_bottom, is_in_viewport + let n_data_items = 1; // bid let post_data_index = 0; for (let j = 0 ; j < n_data_items ; j++) { post_data_index = content.indexOf("_", post_data_index) + 1; @@ -35,7 +35,6 @@ else { elem.removeAttribute("aria-roledescription"); } - } } } diff --git a/core/src/browsergym/core/observation.py b/core/src/browsergym/core/observation.py index 3ea9d8ac..e5e63114 100644 --- a/core/src/browsergym/core/observation.py +++ b/core/src/browsergym/core/observation.py @@ -8,10 +8,16 @@ import re from .constants import BROWSERGYM_ID_ATTRIBUTE as BID_ATTR +from .constants import BROWSERGYM_VISIBILITY_ATTRIBUTE as VIS_ATTR +from .constants import BROWSERGYM_SETOFMARKS_ATTRIBUTE as SOM_ATTR MARK_FRAMES_MAX_TRIES = 3 +class MarkingError(Exception): + pass + + def _pre_extract(page: playwright.sync_api.Page): """ pre-extraction routine, marks dom elements (set bid and dynamic attributes like value and checked) @@ -22,47 +28,41 @@ def _pre_extract(page: playwright.sync_api.Page): # we can't run this loop in JS due to Same-Origin Policy # (can't access the content of an iframe from a another one) - def mark_frames_recursive( - frame, - global_iframe_position, - iframe_offset=None, - ): - # get the bid of the parent frame element - try: - parent_bid = frame.frame_element().get_attribute(BID_ATTR) - except: - parent_bid = "" + def mark_frames_recursive(frame, frame_bid: str): + assert frame_bid == "" or (frame_bid.islower() and frame_bid.isalpha()) + # mark all DOM elements in the frame (it will use the parent frame element's bid as a prefix) - super_iframe_offset = frame.evaluate( + warning_msgs = frame.evaluate( js_frame_mark_elements, - [ - parent_bid, - BID_ATTR, - global_iframe_position, - iframe_offset, - ], + [frame_bid, BID_ATTR], ) + # print warning messages if any + for msg in warning_msgs: + logging.warning(msg) # recursively mark all descendant frames - for _, sub_frame in enumerate(frame.child_frames): - if not sub_frame.is_detached(): - is_frame_hidden = sub_frame.evaluate( - """ () => { - const style = window.getComputedStyle(document.documentElement); - const is_null_size = document.documentElement.offsetWidth <= 0 || document.documentElement.offsetHeight <= 0; - return style.display === 'none' || style.visibility === 'hidden' || is_null_size; -}""" + for child_frame in frame.child_frames: + # deal with detached frames + if child_frame.is_detached(): + continue + # deal with weird frames (pdf viewer in ) + child_frame_elem = child_frame.frame_element() + if not child_frame_elem.content_frame() == child_frame: + logging.warning( + f"Skipping frame '{child_frame.name}' for marking, seems problematic." ) - if not is_frame_hidden: - sub_iframe_position = { - key: sub_frame.frame_element().bounding_box()[key] for key in ["x", "y"] - } - mark_frames_recursive(sub_frame, sub_iframe_position, super_iframe_offset) + continue + # deal with sandboxed frames with blocked script execution + sandbox_attr = child_frame_elem.get_attribute("sandbox") + if sandbox_attr is not None and "allow-scripts" not in sandbox_attr.split(): + continue + child_frame_bid = child_frame_elem.get_attribute(BID_ATTR) + if child_frame_bid is None: + raise MarkingError("Cannot mark a child frame without a bid.") + mark_frames_recursive(child_frame, frame_bid=child_frame_bid) # mark all frames recursively - global_iframe_position = {"x": 0, "y": 0} - - mark_frames_recursive(page.main_frame, global_iframe_position) + mark_frames_recursive(page.main_frame, frame_bid="") def _post_extract(page: playwright.sync_api.Page): @@ -73,6 +73,16 @@ def _post_extract(page: playwright.sync_api.Page): # we can't run this loop in JS due to Same-Origin Policy # (can't access the content of an iframe from a another one) for frame in page.frames: + if not frame == page.main_frame: + # deal with weird frames (pdf viewer in ) + if not frame.frame_element().content_frame() == frame: + logging.warning(f"Skipping frame '{frame.name}' for unmarking, seems problematic.") + continue + # deal with sandboxed frames with blocked script execution + sandbox_attr = frame.frame_element().get_attribute("sandbox") + if sandbox_attr is not None and "allow-scripts" not in sandbox_attr.split(): + continue + try: frame.evaluate(js_frame_unmark_elements) except playwright.sync_api.Error as e: @@ -118,29 +128,11 @@ def extract_screenshot(page: playwright.sync_api.Page): # TODO: handle more data items if needed -__BID_EXPR = r"([-0-9]+)" +__BID_EXPR = r"([a-z0-9]+)" __FLOAT_EXPR = r"([+-]?(?:[0-9]*[.])?[0-9]+)" __BOOL_EXPR = r"([01])" # bid, bbox_left, bbox_top, center_x, center_y, bbox_right, bbox_bottom, is_in_viewport -__DATA_REGEXP = re.compile( - __BID_EXPR - + r"_" - + __FLOAT_EXPR - + r"_" - + __FLOAT_EXPR - + r"_" - + __FLOAT_EXPR - + r"_" - + __FLOAT_EXPR - + r"_" - + __FLOAT_EXPR - + r"_" - + __FLOAT_EXPR - + r"_" - + __BOOL_EXPR - + r"_" - + r"(.*)" -) +__DATA_REGEXP = re.compile(__BID_EXPR + r"_" + r"(.*)") def extract_data_items_from_aria(string): @@ -209,7 +201,7 @@ def extract_dom_snapshot( for node_attributes in document["nodes"]["attributes"]: i = 0 # find the "aria-roledescription" attribute, if any - while i < len(node_attributes): + for i in range(0, len(node_attributes), 2): attr_name_id = node_attributes[i] attr_value_id = node_attributes[i + 1] if attr_name_id == target_attr_name_id: @@ -223,16 +215,179 @@ def extract_dom_snapshot( processed_string_ids.add( attr_value_id ) # mark string as processed (in case several "aria-roledescription" attributes share the same value string) + attr_value = new_attr_value # remove "aria-roledescription" attribute (name and value) if empty - if new_attr_value == "": + if attr_value == "": del node_attributes[i : i + 2] # once "aria-roledescription" is found, exit the search break - i += 2 return dom_snapshot +def extract_dom_extra_properties(dom_snapshot): + def to_string(idx): + if idx == -1: + return None + else: + return dom_snapshot["strings"][idx] + + # pre-locate important string ids + try: + bid_string_id = dom_snapshot["strings"].index(BID_ATTR) + except ValueError: + bid_string_id = -1 + try: + vis_string_id = dom_snapshot["strings"].index(VIS_ATTR) + except ValueError: + vis_string_id = -1 + try: + som_string_id = dom_snapshot["strings"].index(SOM_ATTR) + except ValueError: + som_string_id = -1 + + # build the iframe tree (DFS from the first frame) + doc_properties = { + 0: { + "parent": None, + } + } + + docs_to_process = [0] + while docs_to_process: + doc = docs_to_process.pop(-1) # DFS + + children = dom_snapshot["documents"][doc]["nodes"]["contentDocumentIndex"] + for node, child_doc in zip(children["index"], children["value"]): + doc_properties[child_doc] = { + "parent": { + "doc": doc, # parent frame index + "node": node, # node index within the parent frame + } + } + docs_to_process.append(child_doc) + + # recover the absolute x and y position of the frame node in the parent (if any) + parent = doc_properties[doc]["parent"] + if parent: + parent_doc = parent["doc"] + parent_node = parent["node"] + try: + node_layout_idx = dom_snapshot["documents"][parent_doc]["layout"][ + "nodeIndex" + ].index(parent_node) + except ValueError: + node_layout_idx = -1 + if node_layout_idx >= 0: + node_bounds = dom_snapshot["documents"][parent_doc]["layout"]["bounds"][ + node_layout_idx + ] # can be empty? + # absolute position of parent + relative position of frame node within parent + parent_node_abs_x = doc_properties[parent_doc]["abs_pos"]["x"] + node_bounds[0] + parent_node_abs_y = doc_properties[parent_doc]["abs_pos"]["y"] + node_bounds[1] + else: + parent_node_abs_x = 0 + parent_node_abs_y = 0 + else: + parent_node_abs_x = 0 + parent_node_abs_y = 0 + + # get the frame's absolute position, by adding any scrolling offset if any + doc_properties[doc]["abs_pos"] = { + "x": parent_node_abs_x - dom_snapshot["documents"][doc]["scrollOffsetX"], + "y": parent_node_abs_y - dom_snapshot["documents"][doc]["scrollOffsetY"], + } + + document = dom_snapshot["documents"][doc] + doc_properties[doc]["nodes"] = [ + { + "bid": None, # default value, to be filled (str) + "visibility": None, # default value, to be filled (float) + "bbox": None, # default value, to be filled (list) + "clickable": False, # default value, to be filled (bool) + "set_of_marks": None, # default value, to be filled (bool) + } + for _ in enumerate(document["nodes"]["parentIndex"]) + ] # all nodes in document + + # extract clickable property + for node_idx in document["nodes"]["isClickable"]["index"]: + doc_properties[doc]["nodes"][node_idx]["clickable"] = True + + # extract bid and visibility properties (attribute-based) + for node_idx, node_attrs in enumerate(document["nodes"]["attributes"]): + i = 0 + # loop over all attributes + for i in range(0, len(node_attrs), 2): + name_string_id = node_attrs[i] + value_string_id = node_attrs[i + 1] + if name_string_id == bid_string_id: + doc_properties[doc]["nodes"][node_idx]["bid"] = to_string(value_string_id) + if name_string_id == vis_string_id: + doc_properties[doc]["nodes"][node_idx]["visibility"] = float( + to_string(value_string_id) + ) + if name_string_id == som_string_id: + doc_properties[doc]["nodes"][node_idx]["set_of_marks"] = ( + to_string(value_string_id) == "1" + ) + + # extract bbox property (in absolute coordinates) + for node_idx, bounds, client_rect in zip( + document["layout"]["nodeIndex"], + document["layout"]["bounds"], + document["layout"]["clientRects"], + ): + # empty clientRect means element is not actually rendered + if not client_rect: + doc_properties[doc]["nodes"][node_idx]["bbox"] = None + else: + # bounds gives the relative position within the document + doc_properties[doc]["nodes"][node_idx]["bbox"] = bounds.copy() + # adjust for absolute document position + doc_properties[doc]["nodes"][node_idx]["bbox"][0] += doc_properties[doc]["abs_pos"][ + "x" + ] + doc_properties[doc]["nodes"][node_idx]["bbox"][1] += doc_properties[doc]["abs_pos"][ + "y" + ] + + # Note: other interesting fields + # document["nodes"]["parentIndex"] # parent node + # document["nodes"]["nodeType"] + # document["nodes"]["nodeName"] + # document["nodes"]["nodeValue"] + # document["nodes"]["textValue"] + # document["nodes"]["inputValue"] + # document["nodes"]["inputChecked"] + # document["nodes"]["optionSelected"] + # document["nodes"]["pseudoType"] + # document["nodes"]["pseudoIdentifier"] + # document["nodes"]["isClickable"] + # document["textBoxes"] + # document["layout"]["nodeIndex"] + # document["layout"]["bounds"] + # document["layout"]["offsetRects"] + # document["layout"]["scrollRects"] + # document["layout"]["clientRects"] + # document["layout"]["paintOrders"] + + # collect the extra properties of all nodes with a browsergym_id attribute + extra_properties = {} + for doc in doc_properties.keys(): + for node in doc_properties[doc]["nodes"]: + bid = node["bid"] + if bid: + if bid in extra_properties: + logging.warning(f"duplicate {BID_ATTR}={repr(bid)} attribute detected") + extra_properties[bid] = { + extra_prop: node[extra_prop] + for extra_prop in ("visibility", "bbox", "clickable", "set_of_marks") + } + + return extra_properties + + def extract_all_frame_axtrees(page: playwright.sync_api.Page): """ Extracts the AXTree of all frames (main document and iframes) of a Playwright page using Chrome DevTools Protocol. @@ -289,16 +444,7 @@ def extract_all_frame_axtrees(page: playwright.sync_api.Page): del node["properties"][i] # add all extracted "browsergym" properties to the AXTree if data_items: - ( - browsergym_id, - left, - top, - center_x, - center_y, - right, - bottom, - is_in_viewport, - ) = data_items + (browsergym_id,) = data_items node["properties"].append( { "name": "browsergym_id", @@ -308,38 +454,6 @@ def extract_all_frame_axtrees(page: playwright.sync_api.Page): }, } ) - node["properties"].append( - { - "name": "browsergym_center", - "value": { - "type": "list", - "value": (float(center_x), float(center_y)), - }, - } - ) - node["properties"].append( - { - "name": "browsergym_bounding_box", - "value": { - "type": "list", - "value": ( - float(left), - float(top), - float(right), - float(bottom), - ), - }, - } - ) - node["properties"].append( - { - "name": "browsergym_is_in_viewport", - "value": { - "type": "boolean", - "value": bool(is_in_viewport == "1"), - }, - } - ) return frame_axtrees diff --git a/core/src/browsergym/core/registration.py b/core/src/browsergym/core/registration.py index a8c3350d..dd0e36ed 100644 --- a/core/src/browsergym/core/registration.py +++ b/core/src/browsergym/core/registration.py @@ -5,18 +5,24 @@ from .task import AbstractBrowserTask -def register_task(id: str, task_class: Type[AbstractBrowserTask], *args, **kwargs): +def register_task( + id: str, task_class: Type[AbstractBrowserTask], nondeterministic: bool = True, *args, **kwargs +): """ Registers a browser task as a gym environment with its unique id. Args: - task: the task class to register. + id: the id of the task to register (will be prepended by "browsergym/"). + task_class: the task class to register. + nondeterministic: whether the task cannot be guaranteed deterministic transitions. + *args: additional arguments for the browsergym environment. + *kwargs: additional arguments for the browsergym environment. """ gym.register( id=f"browsergym/{id}", - entry_point=lambda *args, **kwargs: BrowserEnv(task_class, *args, **kwargs), - nondeterministic=task_class.nondeterministic, + entry_point=lambda *env_args, **env_kwargs: BrowserEnv(task_class, *env_args, **env_kwargs), + nondeterministic=nondeterministic, *args, **kwargs, ) diff --git a/core/src/browsergym/core/spaces.py b/core/src/browsergym/core/spaces.py index f5e1f005..338dc86d 100644 --- a/core/src/browsergym/core/spaces.py +++ b/core/src/browsergym/core/spaces.py @@ -2,6 +2,7 @@ from typing import Any +from matplotlib.pylab import Generator import numpy as np from gymnasium.spaces import Space, Box, Text from gymnasium.spaces.utils import flatdim, flatten, flatten_space, unflatten @@ -77,3 +78,42 @@ def __repr__(self) -> str: def __eq__(self, other: Any) -> bool: """Check whether ``other`` is equivalent to this instance.""" return isinstance(other, AnyDict) + + +class AnyBox(Space[NDArray[Any]]): + """A space representing an arbitrary dictionary object.""" + + def __init__(self, low, high, shape, dtype): + super().__init__(shape, dtype) + self.low = low + self.high = high + + def contains(self, x: Any) -> bool: + """Return boolean specifying if x is a valid member of this space.""" + if not isinstance(x, np.ndarray): + try: + x = np.asarray(x, dtype=self.dtype) + except (ValueError, TypeError): + return False + + return bool( + np.can_cast(x.dtype, self.dtype) + and len(x.shape) == len(self.shape) + and all([dim in (xdim, -1) for xdim, dim in zip(x.shape, self.shape)]) + and np.all(x >= self.low) + and np.all(x <= self.high) + ) + + def __repr__(self) -> str: + """Gives a string representation of this space.""" + return f"AnyBox(low={repr(self.low)}, high={repr(self.high)}, shape={repr(self.shape)}, dtype={repr(self.dtype)})" + + def __eq__(self, other: Any) -> bool: + """Check whether ``other`` is equivalent to this instance.""" + return ( + isinstance(other, AnyBox) + and self.low == other.low + and self.high == other.high + and self.shape == other.shape + and self.dtype == other.dtype + ) diff --git a/core/src/browsergym/core/task.py b/core/src/browsergym/core/task.py index e3ad8720..6555223d 100644 --- a/core/src/browsergym/core/task.py +++ b/core/src/browsergym/core/task.py @@ -1,3 +1,4 @@ +import numpy as np import playwright.sync_api from abc import ABC, abstractmethod @@ -10,21 +11,27 @@ class AbstractBrowserTask(ABC): """ - # gym metadata (default values, can be overloaded) - nondeterministic: bool = True - @classmethod @abstractmethod def get_task_id(cls): pass + def __init__(self, seed: int) -> None: + # initiate a random number generator + self.random = np.random.RandomState(seed) + + # task properties, will be used to set up the browsergym environment + # default values, can be overriden in children classes + self.viewport = {"width": 1280, "height": 720} + self.slow_mo = 1000 # ms + self.timeout = 5000 # ms + @abstractmethod - def setup(self, seed: int, page: playwright.sync_api.Page) -> tuple[str, dict]: + def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]: """ Set up everything needed to execute the task. Args: - seed: a seed for the task's randomness. page: the active playwright page. Returns: @@ -71,17 +78,19 @@ class OpenEndedTask(AbstractBrowserTask): def get_task_id(cls): return "openended" - def __init__(self, start_url: str, goal: str = None) -> None: + def __init__(self, seed: int, start_url: str, goal: str = None) -> None: """ Args: + seed: random seed. start_url: str, the url for the starting page. goal: str, the initial goal. """ + super().__init__(seed) self.start_url = start_url self.goal = goal - def setup(self, seed: int, page: playwright.sync_api.Page) -> tuple[str, dict]: + def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]: page.goto(self.start_url, timeout=10000) return self.goal, {} diff --git a/core/src/browsergym/utils/obs.py b/core/src/browsergym/utils/obs.py index 795179ee..f63ff2b7 100644 --- a/core/src/browsergym/utils/obs.py +++ b/core/src/browsergym/utils/obs.py @@ -1,8 +1,16 @@ import ast +import numpy as np +import PIL.Image +import PIL.ImageDraw +import PIL.ImageFont from collections import defaultdict from bs4 import BeautifulSoup +from browsergym.core.constants import BROWSERGYM_ID_ATTRIBUTE as BID_ATTR +from browsergym.core.constants import BROWSERGYM_VISIBILITY_ATTRIBUTE as VIS_ATTR +from browsergym.core.constants import BROWSERGYM_SETOFMARKS_ATTRIBUTE as SOM_ATTR + IGNORED_AXTREE_ROLES = ["LineBreak"] IGNORED_AXTREE_PROPERTIES = ( @@ -17,104 +25,139 @@ def flatten_dom_to_str( - DOM_tree, + dom_snapshot, + extra_properties: dict = None, with_visible: bool = False, + with_clickable: bool = False, with_center_coords: bool = False, with_bounding_box_coords: bool = False, + with_som: bool = False, filter_visible_only: bool = False, + filter_with_bid_only: bool = False, + filter_som_only: bool = False, coord_decimals: int = 0, ) -> str: """Formats a DOM snapshot into a string text""" - coord_format = f":0.{coord_decimals}f" + def to_string(idx): + if idx == -1: + return None + else: + return dom_snapshot["strings"][idx] - def parse_DOM(document_idx) -> str: - # adopted from [natbot](https://github.com/nat/natbot) + def parse_document(document_idx) -> str: + # adapted from [natbot](https://github.com/nat/natbot) - strings = DOM_tree["strings"] - nodes = DOM_tree["documents"][document_idx]["nodes"] - node_iframe_link = nodes["contentDocumentIndex"] - graph = defaultdict(lambda: []) + nodes = dom_snapshot["documents"][document_idx]["nodes"] + node_children = defaultdict(lambda: []) for node_idx in range(len(nodes["nodeName"])): parent_idx = nodes["parentIndex"][node_idx] if parent_idx != -1: - graph[parent_idx].append(node_idx) - - def dfs(idx: int) -> str: - node_name = strings[nodes["nodeName"][idx]] - can_skip = ( - "#" in node_name or "::" in node_name or node_name == "html" - ) # We skip the root node - node_name = node_name.lower().strip() - html = "" - - # print node opening tag - if not can_skip: - html += f"<{node_name}" - node_attr_idxs = nodes["attributes"][idx] - if node_attr_idxs: - for i in range(0, len(node_attr_idxs), 2): - attr_name = strings[node_attr_idxs[i]] - - # filter visible elements if requested - if ( - filter_visible_only - and attr_name == "browsergym_is_in_viewport" - and strings[node_attr_idxs[i + 1]] == "0" - ): - can_skip = True - break - - # print browsergym attributes if requested (with new names) - if attr_name == "browsergym_is_in_viewport": - if with_visible: - attr_value = strings[node_attr_idxs[i + 1]] - html += f' is_visible="{attr_value}"' - elif attr_name == "browsergym_center": - if with_center_coords: - attr_value = strings[node_attr_idxs[i + 1]] - html += f' center="{_get_coord_str(attr_value, coord_decimals)}"' - - elif attr_name == "browsergym_bounding_box": - if with_bounding_box_coords: - attr_value = strings[node_attr_idxs[i + 1]] - html += f' box="{_get_coord_str(attr_value, coord_decimals)}"' - - # print other attributes + node_children[parent_idx].append(node_idx) + + def dfs(node_idx: int, parent_node_skipped: bool) -> str: + + # https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType + # https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeName + # https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeValue + + node_type = nodes["nodeType"][node_idx] + node_name = to_string(nodes["nodeName"][node_idx]) + node_value = to_string(nodes["nodeValue"][node_idx]) + html_before = "" + html_after = "" + skip_node = False + + # text nodes: print text content only if parent was not skipped + if node_type == 3: # node_name == "#text" + if not parent_node_skipped and node_value is not None: + html_before += node_value + + # CData nodes: print content only if parent was not skipped + elif node_type == 4: # node_name == "#cdata-section": + if not parent_node_skipped and node_value is not None: + html_before += f"" + + # processing instructions, comments, documents, doctypes, document fragments: don't print + elif node_type in (7, 8, 9, 10, 11): + skip_node = True + + # now we should have an element node + else: + assert node_type == 1 + + tag_name = node_name.lower().strip() + attributes = [] # to be printed as attributes with the tag + bid = None + + # parse node attributes + node_attr_idxs = nodes["attributes"][node_idx] + for i in range(0, len(node_attr_idxs), 2): + attr_name = to_string(node_attr_idxs[i]) + attr_value = to_string(node_attr_idxs[i + 1]) + + # extract and print bid + if attr_name == BID_ATTR: + bid = attr_value + # ignore browsergym attributes + elif attr_name in (VIS_ATTR, SOM_ATTR): + pass + # print other attributes + else: + if attr_value is None: + # attribute value missing + attributes.append(f"{attr_name}") else: - if node_attr_idxs[i + 1] >= 0: - attr_value = strings[node_attr_idxs[i + 1]] - # attribute value present - html += f' {attr_name}="{attr_value}"' - else: - # attribute value missing - html += f" {attr_name}" - - html += f">" - if can_skip: - html = "" - # print inner text - node_value_idx = nodes["nodeValue"][idx] - if node_value_idx >= 0: - html += " ".join(strings[node_value_idx].split()) + # attribute value present + attributes.append(f'{attr_name}="{attr_value}"') + + skip_node, extra_attributes_to_print = _process_bid( + bid, + extra_properties=extra_properties, + with_visible=with_visible, + with_clickable=with_clickable, + with_center_coords=with_center_coords, + with_bounding_box_coords=with_bounding_box_coords, + with_som=with_som, + filter_visible_only=filter_visible_only, + filter_with_bid_only=filter_with_bid_only, + filter_som_only=filter_som_only, + coord_decimals=coord_decimals, + ) + + # insert extra attributes before regular attributes + attributes = extra_attributes_to_print + attributes + + # insert bid as first attribute + if bid is not None: + attributes.insert(0, f'bid="{bid}"') + + if not skip_node: + # print node opening tag, with its attributes + html_before += f"<{tag_name}" + " ".join([""] + attributes) + ">" + # print node closing tag + html_after += f"" + + html = "" + html += html_before # recursively print iframe nodes if any - if idx in node_iframe_link["index"]: - sub_document_idx = node_iframe_link["value"][node_iframe_link["index"].index(idx)] - html += parse_DOM(document_idx=sub_document_idx) + if node_idx in nodes["contentDocumentIndex"]["index"]: + sub_document_idx = nodes["contentDocumentIndex"]["value"][ + nodes["contentDocumentIndex"]["index"].index(node_idx) + ] + html += parse_document(document_idx=sub_document_idx) - # recursively print children nodes - for child_idx in graph[idx]: - html += dfs(child_idx) + # recursively print children nodes if any + for child_idx in node_children[node_idx]: + html += dfs(node_idx=child_idx, parent_node_skipped=skip_node) - # print node closing tag - if not can_skip: - html += f"" + html += html_after return html - html = dfs(idx=0) + html = dfs(node_idx=0, parent_node_skipped=False) # Format the HTML document with indentation soup = BeautifulSoup(html, "lxml") @@ -122,7 +165,7 @@ def dfs(idx: int) -> str: return html - html = parse_DOM(0) + html = parse_document(document_idx=0) return html @@ -149,96 +192,305 @@ def _get_coord_str(coord, decimals): return f"({coord_str})" +def _process_bid( + bid, + extra_properties: dict = None, + with_visible: bool = False, + with_clickable: bool = False, + with_center_coords: bool = False, + with_bounding_box_coords: bool = False, + with_som: bool = False, + filter_visible_only: bool = False, + filter_with_bid_only: bool = False, + filter_som_only: bool = False, + coord_decimals: int = 0, +): + """ + Process extra attributes and attribute-based filters, for the element with the given bid. + + Returns: + A flag indicating if the element should be skipped or not (due to filters). + Attributes to be printed, as a list of "x=y" strings. + """ + + if extra_properties is None: + if any( + ( + with_visible, + with_clickable, + with_center_coords, + with_bounding_box_coords, + with_som, + filter_visible_only, + filter_with_bid_only, + filter_som_only, + ) + ): + raise ValueError("extra_properties argument required") + else: + extra_properties = {} + + skip_element = False + attributes_to_print = [] + + if bid is None: + # skip nodes without a bid (if requested) + if filter_with_bid_only: + skip_element = True + if filter_som_only: + skip_element = True + if filter_visible_only: + # element without bid have no visibility mark, they could be visible or non-visible + pass # TODO: we consider them as visible. Is this what we want? + + # parse extra browsergym properties, if node has a bid + else: + if bid in extra_properties: + node_vis = extra_properties[bid]["visibility"] + node_bbox = extra_properties[bid]["bbox"] + node_is_clickable = extra_properties[bid]["clickable"] + node_in_som = extra_properties[bid]["set_of_marks"] + node_is_visible = node_vis >= 0.5 + # skip non-visible nodes (if requested) + if filter_visible_only and not node_is_visible: + skip_element = True + if filter_som_only and not node_in_som: + skip_element = True + # print extra attributes if requested (with new names) + if with_som and node_in_som: + attributes_to_print.insert(0, f'som="{int(node_in_som)}"') + if with_visible: + attributes_to_print.insert(0, f'visible="{int(node_is_visible)}"') + if with_clickable and node_is_clickable: + attributes_to_print.insert(0, f'clickable="{int(node_is_clickable)}"') + if with_center_coords and node_bbox is not None: + x, y, width, height = node_bbox + center = (x + width / 2, y + height / 2) + attributes_to_print.insert(0, f'center="{_get_coord_str(center, coord_decimals)}"') + if with_bounding_box_coords and node_bbox is not None: + x, y, width, height = node_bbox + box = (x, y, x + width, y + height) + attributes_to_print.insert(0, f'box="{_get_coord_str(box, coord_decimals)}"') + + return skip_element, attributes_to_print + + def flatten_axtree_to_str( AX_tree, + extra_properties: dict = None, with_visible: bool = False, + with_clickable: bool = False, with_center_coords: bool = False, with_bounding_box_coords: bool = False, + with_som: bool = False, filter_visible_only: bool = False, + filter_with_bid_only: bool = False, + filter_som_only: bool = False, coord_decimals: int = 0, - ignore_roles=IGNORED_AXTREE_ROLES, + ignored_roles=IGNORED_AXTREE_ROLES, ignored_properties=IGNORED_AXTREE_PROPERTIES, - remove_rdundant_static_text: bool = True, + remove_redundant_static_text: bool = True, ) -> str: """Formats the accessibility tree into a string text""" node_id_to_idx = {} for idx, node in enumerate(AX_tree["nodes"]): node_id_to_idx[node["nodeId"]] = idx - def dfs(idx: int, depth: int) -> str: + def dfs(node_idx: int, depth: int, parent_node_filtered: bool) -> str: tree_str = "" - node = AX_tree["nodes"][idx] + node = AX_tree["nodes"][node_idx] indent = "\t" * depth - valid_node = True - role = node["role"]["value"] + skip_node = False + filter_node = False + node_role = node["role"]["value"] - if role in ignore_roles: + if node_role in ignored_roles: + skip_node = True pass elif "name" not in node: + skip_node = True pass else: - print_node = True - name = node["name"]["value"] - node_str = f"{role} {repr(name.strip())}" - + node_name = node["name"]["value"] if "value" in node and "value" in node["value"]: - node_str += f' value: {repr(node["value"]["value"])}' + node_value = node["value"]["value"] + else: + node_value = None - properties = [] + attributes = [] + bid = None for property in node.get("properties", []): if not "value" in property: continue if not "value" in property["value"]: continue - prop_name, value = property["name"], property["value"]["value"] + prop_name = property["name"] + prop_value = property["value"]["value"] + if prop_name == "browsergym_id": - node_str = f"[{value}] " + node_str - elif prop_name == "browsergym_center": - if with_center_coords: - coord_str = _get_coord_str(value, coord_decimals) - node_str += f", center={coord_str}" - elif prop_name == "browsergym_bounding_box": - if with_bounding_box_coords: - coord_str = _get_coord_str(value, coord_decimals) - node_str += f", box={coord_str}" - elif prop_name == "browsergym_is_in_viewport": - # filter visible elements if requested - if filter_visible_only and not value: - print_node = False - break - if with_visible: - visible_str = "visible" if value else "hidden" - node_str += f", {visible_str}" + bid = prop_value + elif prop_name in ignored_properties: + continue elif prop_name in ("required", "focused", "atomic"): - if value: - properties.append(prop_name) - elif prop_name not in ignored_properties: - properties.append(f"{prop_name}={repr(value)}") + if prop_value: + attributes.append(prop_name) + else: + attributes.append(f"{prop_name}={repr(prop_value)}") + + if node_role == "generic" and not attributes: + skip_node = True + + if node_role == "StaticText": + if parent_node_filtered: + skip_node = True + else: + filter_node, extra_attributes_to_print = _process_bid( + bid, + extra_properties=extra_properties, + with_visible=with_visible, + with_clickable=with_clickable, + with_center_coords=with_center_coords, + with_bounding_box_coords=with_bounding_box_coords, + with_som=with_som, + filter_visible_only=filter_visible_only, + filter_with_bid_only=filter_with_bid_only, + filter_som_only=filter_som_only, + coord_decimals=coord_decimals, + ) + + # if either is True, skip the node + skip_node = skip_node or filter_node + + # insert extra attributes before regular attributes + attributes = extra_attributes_to_print + attributes + + # actually print the node string + if not skip_node: + node_str = f"{node_role} {repr(node_name.strip())}" + + if bid is not None: + node_str = f"[{bid}] " + node_str + + if node_value is not None: + node_str += f' value={repr(node["value"]["value"])}' + + if attributes: + node_str += ", ".join([""] + attributes) - if role in ["generic"] and not properties: - print_node = False - - if properties: - node_str += " " + ", ".join(properties) - - if print_node: tree_str += f"{indent}{node_str}" - for _, child_node_id in enumerate(node["childIds"]): + for child_node_id in node["childIds"]: if child_node_id not in node_id_to_idx or child_node_id == node["nodeId"]: continue # mark this to save some tokens - child_depth = depth + 1 if valid_node else depth - child_str = dfs(node_id_to_idx[child_node_id], child_depth) - if child_str.strip(): - if tree_str.strip(): + child_depth = depth if skip_node else (depth + 1) + child_str = dfs( + node_id_to_idx[child_node_id], child_depth, parent_node_filtered=filter_node + ) + if child_str: + if tree_str: tree_str += "\n" tree_str += child_str return tree_str - tree_str = dfs(0, 0) - if remove_rdundant_static_text: + tree_str = dfs(0, 0, False) + if remove_redundant_static_text: tree_str = _remove_redundant_static_text(tree_str) return tree_str + + +def overlay_som( + screenshot: np.typing.ArrayLike, + extra_properties: dict, + fontsize: int = 12, + linewidth: int = 2, + tag_margin: int = 2, +): + img = PIL.Image.fromarray(screenshot).copy() # make a copy + img = img.convert(mode="RGBA") + draw = PIL.ImageDraw.Draw(img) + + font = PIL.ImageFont.load_default(size=fontsize) + + # https://stackoverflow.com/questions/51908563/dotted-or-dashed-line-with-python-pillow/58885306#58885306 + import math # math has the fastest sqrt + + def linedashed(draw: PIL.ImageDraw.Draw, x0, y0, x1, y1, fill, width, dashlen=4, ratio=3): + dx = x1 - x0 # delta x + dy = y1 - y0 # delta y + # check whether we can avoid sqrt + if dy == 0: + vlen = dx + elif dx == 0: + vlen = dy + else: + vlen = math.sqrt(dx * dx + dy * dy) # length of line + xa = dx / vlen # x add for 1px line length + ya = dy / vlen # y add for 1px line length + step = dashlen * ratio # step to the next dash + a0 = 0 + while a0 < vlen: + a1 = a0 + dashlen + if a1 > vlen: + a1 = vlen + draw.line( + (x0 + xa * a0, y0 + ya * a0, x0 + xa * a1, y0 + ya * a1), fill=fill, width=width + ) + a0 += step + + for bid, properties in extra_properties.items(): + if properties["set_of_marks"] and properties["bbox"]: + x, y, width, height = properties["bbox"] + + # draw bounding box with dashed lines + linedashed(draw, x, y, x + width, y, fill=(0, 0, 0, 255), width=linewidth) + linedashed( + draw, x + width, y, x + width, y + height, fill=(0, 0, 0, 255), width=linewidth + ) + linedashed( + draw, x, y + height, x + width, y + height, fill=(0, 0, 0, 255), width=linewidth + ) + linedashed(draw, x, y, x, y + height, fill=(0, 0, 0, 255), width=linewidth) + + # get text box size (left, top, right, bottom) + tag_box = font.getbbox( + bid, + ) + + # set tag size, including margins + tag_size = ( + (tag_box[2] - tag_box[0] + 2 * (tag_margin + 1)), + (tag_box[3] - tag_box[1] + 2 * (tag_margin + 1)), + ) + + # create tag image with correct size and black background + tag_img = PIL.Image.new("RGBA", tag_size, "black") + tag_draw = PIL.ImageDraw.Draw(tag_img) + # write text with 1px horizontal margin + tag_draw.text( + (-tag_box[0] + tag_margin + 1, -tag_box[1] + tag_margin + 1), + bid, + font=font, + fill=(255, 255, 255, 255), + spacing=0, + ) + tag_draw.rectangle( + (0, 0, tag_size[0] - 1, tag_size[1] - 1), + fill=None, + outline=(255, 255, 255, 255), + width=1, + ) + + # draw tag in the source image, upper left of the bounding box + tag_pos = (x + 0, y - tag_size[1] / 2 + 4) + tag_pos = list(map(round, tag_pos)) + img.paste(tag_img, tag_pos) + + # convert to RGB (3 channels) + img = img.convert(mode="RGB") + # convert to a numpy array + img = np.array(img) + + return img diff --git a/core/tests/data/test_page_2.html b/core/tests/data/test_page_2.html new file mode 100644 index 00000000..250a8170 --- /dev/null +++ b/core/tests/data/test_page_2.html @@ -0,0 +1,63 @@ + + + + + Simple Form + + + +

Simple Form

+ +
+ +

+ + +

+ + +

+ +
+

+ + +

+ + + +
+ + + Text within in non-html tag + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

Text that should not be visible

+ + + diff --git a/core/tests/test_actions_highlevel.py b/core/tests/test_actions_highlevel.py index b1723048..779746c2 100644 --- a/core/tests/test_actions_highlevel.py +++ b/core/tests/test_actions_highlevel.py @@ -97,7 +97,7 @@ def test_valid_action(): env = gym.make( "browsergym/openended", - start_url=CHECKBOX_URL, + task_kwargs={"start_url": CHECKBOX_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -288,7 +288,7 @@ def test_invalid_action(): env = gym.make( "browsergym/openended", - start_url=CHECKBOX_URL, + task_kwargs={"start_url": CHECKBOX_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -304,7 +304,7 @@ def test_invalid_action(): obs, reward, term, trunc, info = env.step(action) # error - assert "TimeoutError" in obs["last_action_error"] + assert "ValueError" in obs["last_action_error"] # invalid bid value type action = f"""\ @@ -450,7 +450,7 @@ def test_click_through_frames(): env = gym.make( "browsergym/openended", - start_url=MULTI_IFRAME_URL, + task_kwargs={"start_url": MULTI_IFRAME_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -490,7 +490,7 @@ def test_fill_through_iframe(): env = gym.make( "browsergym/openended", - start_url=MULTI_IFRAME_URL, + task_kwargs={"start_url": MULTI_IFRAME_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -534,7 +534,7 @@ def test_click(): env = gym.make( "browsergym/openended", - start_url=CHECKBOX_URL, + task_kwargs={"start_url": CHECKBOX_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -606,7 +606,7 @@ def test_hover(): env = gym.make( "browsergym/openended", - start_url=HOVER_URL, + task_kwargs={"start_url": HOVER_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -651,7 +651,7 @@ def test_fill_type_press(): action_set = HighLevelActionSet(subsets=["bid", "coord"]) env = gym.make( "browsergym/openended", - start_url=TEXT_INPUT_URL, + task_kwargs={"start_url": TEXT_INPUT_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -806,7 +806,7 @@ def test_key_press(): env = gym.make( "browsergym/openended", - start_url=TEXT_INPUT_URL, + task_kwargs={"start_url": TEXT_INPUT_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -848,7 +848,7 @@ def test_goto(): env = gym.make( "browsergym/openended", - start_url=url1, + task_kwargs={"start_url": url1}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -896,26 +896,22 @@ def test_scroll(): env = gym.make( "browsergym/openended", - start_url=LONG_PAGE_URL, - headless=False, + task_kwargs={"start_url": LONG_PAGE_URL}, + headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, action_mapping=action_set.to_python_code, ) def extract_coords_from_elem(elem): - x, y = map( - float, - re.search( - r"\(([-+]?[0-9\.]+),[\s]*([-+]?[0-9\.]+)\)", - elem.get("center"), - ).groups(), - ) - return x, y + return ast.literal_eval(elem.get("center")) def get_top_bottom_elems(obs): soup = bs4.BeautifulSoup( - flatten_dom_to_str(obs["dom_object"], with_center_coords=True), "lxml" + flatten_dom_to_str( + obs["dom_object"], obs["extra_element_properties"], with_center_coords=True + ), + "lxml", ) top = soup.find("input", attrs={"type": "checkbox", "id": "top"}) bottom = soup.find("input", attrs={"type": "checkbox", "id": "bottom"}) @@ -1009,7 +1005,7 @@ def get_top_bottom_elems(obs): # def test_meta_action(): # env = BrowserEnv( # task_entrypoint=OpenEndedTask, -# start_url=TEXT_INPUT_URL, +# task_kwargs={"start_url": TEXT_INPUT_URL}, # headless=__HEADLESS__, # ) # obs, info = env.reset() @@ -1143,7 +1139,7 @@ def get_top_bottom_elems(obs): # def test_clear_success(): # env = BrowserEnv( # task_entrypoint=OpenEndedTask, -# start_url=TEXT_INPUT_URL, +# task_kwargs={"start_url": TEXT_INPUT_URL}, # headless=__HEADLESS__, # ) # obs, info = env.reset() @@ -1213,7 +1209,7 @@ def get_top_bottom_elems(obs): # """In this test, we try to build a ClearAction but we use invalid args, and we check that the action fails when executed in the environment""" # env = BrowserEnv( # task_entrypoint=OpenEndedTask, -# start_url=TEXT_INPUT_URL, +# task_kwargs={"start_url": TEXT_INPUT_URL}, # headless=__HEADLESS__, # ) # obs, info = env.reset() @@ -1307,7 +1303,7 @@ def test_mouse_down_up(): env = gym.make( "browsergym/openended", - start_url=CHECKBOX_URL, + task_kwargs={"start_url": CHECKBOX_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -1316,7 +1312,10 @@ def test_mouse_down_up(): def get_checkbox_elem(obs): soup = bs4.BeautifulSoup( - flatten_dom_to_str(obs["dom_object"], with_center_coords=True), "lxml" + flatten_dom_to_str( + obs["dom_object"], obs["extra_element_properties"], with_center_coords=True + ), + "lxml", ) checkbox = soup.find("input", attrs={"type": "checkbox", "id": "vehicle1"}) return checkbox diff --git a/core/tests/test_gym_envs.py b/core/tests/test_gym_envs.py index 0ebea442..968cf525 100644 --- a/core/tests/test_gym_envs.py +++ b/core/tests/test_gym_envs.py @@ -23,7 +23,7 @@ __DATA_DIR = pathlib.Path(__file__).resolve().parent / "data" TEST_PAGE = f"file://{__DATA_DIR}/test_page.html" BASIC_IFRAME_PAGE = f"file://{__DATA_DIR}/basic_iframe_site/basic_iframe_2.html" -DEMO_MODES = ["default", "only_visible_elements"] +DEMO_MODES = ["default", "only_visible_elements", "all_blue"] def test_gym_env(): @@ -31,7 +31,7 @@ def test_gym_env(): env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -92,7 +92,7 @@ def test_max_episode_steps(): # no max_steps env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -112,7 +112,7 @@ def test_max_episode_steps(): # max_steps = 2 env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -137,7 +137,7 @@ def test_active_page(): action_set = PythonActionSet() env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -191,11 +191,10 @@ def test_nested_iframes_default_demo_mode(): action_set = HighLevelActionSet(demo_mode=demo_mode) env = gym.make( "browsergym/openended", - start_url=BASIC_IFRAME_PAGE, + task_kwargs={"start_url": BASIC_IFRAME_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, - demo_mode=demo_mode, action_mapping=action_set.to_python_code, ) obs, info = env.reset() @@ -228,12 +227,11 @@ def test_demo_mode(demo_mode): action_set = HighLevelActionSet(demo_mode=demo_mode) env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, action_mapping=action_set.to_python_code, - demo_mode=demo_mode, ) obs, info = env.reset() assert not obs["last_action_error"] @@ -282,3 +280,23 @@ def test_demo_mode(demo_mode): assert checkbox.has_attr("checked") env.close() + + +@pytest.mark.parametrize("resizeable_window", (True, False)) +@pytest.mark.parametrize("size", ((1600, 1200), (800, 800))) +def test_resizeable_window(resizeable_window, size): + env = gym.make( + "browsergym/openended", + task_kwargs={"start_url": TEST_PAGE}, + headless=__HEADLESS, + slow_mo=__SLOW_MO, + timeout=__TIMEOUT, + viewport={"width": size[0], "height": size[1]}, + resizeable_window=resizeable_window, + ) + obs, info = env.reset() + assert not obs["last_action_error"] + + assert (obs["screenshot"].shape[1], obs["screenshot"].shape[0]) == size + + env.close() diff --git a/core/tests/test_observation.py b/core/tests/test_observation.py index 2d594752..a488dbc0 100644 --- a/core/tests/test_observation.py +++ b/core/tests/test_observation.py @@ -17,7 +17,7 @@ ) # bugfix: use same playwright instance in browsergym and pytest - +from utils import setup_playwright from browsergym.core.observation import ( _pre_extract, @@ -37,6 +37,7 @@ __DATA_DIR = Path(__file__).resolve().parent / "data" TEST_PAGE = f"file://{__DATA_DIR}/test_page.html" +TEST_PAGE_2 = f"file://{__DATA_DIR}/test_page_2.html" MULTI_IFRAME_URL = f"file://{__DATA_DIR}/basic_iframe_site/basic_iframe_2.html" SHADOW_DOM_URL = f"file://{__DATA_DIR}/basic_shadow_dom_site/basic_shadow_dom.html" SIMPLE_SHADOW_DOM_URL = f"file://{__DATA_DIR}/basic_shadow_dom_site/simple_shadow_dom.html" @@ -52,7 +53,7 @@ def test_extract_screenshot(): env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -78,7 +79,7 @@ def test_extract_screenshot(): def test_extract_axtree_simple(): env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -101,7 +102,7 @@ def test_extract_axtree_simple(): def test_extract_axtree_multi_iframe(): env = gym.make( "browsergym/openended", - start_url=MULTI_IFRAME_URL, + task_kwargs={"start_url": MULTI_IFRAME_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -130,7 +131,7 @@ def test_extract_axtree_multi_iframe(): def test_extract_dom_simple(): env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -151,7 +152,7 @@ def test_extract_dom_simple(): def test_extract_dom_multi_iframe(): env = gym.make( "browsergym/openended", - start_url=MULTI_IFRAME_URL, + task_kwargs={"start_url": MULTI_IFRAME_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -172,7 +173,7 @@ def test_extract_dom_multi_iframe(): def test_simple_shadowdom(): env = gym.make( "browsergym/openended", - start_url=SIMPLE_SHADOW_DOM_URL, + task_kwargs={"start_url": SIMPLE_SHADOW_DOM_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -206,7 +207,7 @@ def test_simple_shadowdom(): def test_nested_shadowdom(): env = gym.make( "browsergym/openended", - start_url=SHADOW_DOM_URL, + task_kwargs={"start_url": SHADOW_DOM_URL}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -252,7 +253,7 @@ def test_nested_shadowdom(): def test_dom_has_bids_no_aria(url): env = gym.make( "browsergym/openended", - start_url=url, + task_kwargs={"start_url": url}, headless=__HEADLESS, slow_mo=__SLOW_MO, viewport=__VIEWPORT, @@ -325,7 +326,7 @@ def test_dom_has_bids_no_aria(url): def test_dom_to_text(): env = gym.make( "browsergym/openended", - start_url=TEST_PAGE, + task_kwargs={"start_url": TEST_PAGE_2}, headless=__HEADLESS, slow_mo=__SLOW_MO, timeout=__TIMEOUT, @@ -354,13 +355,58 @@ def test_dom_to_text(): assert "Janice" in dom assert "janice@mail.com" in dom + dom = flatten_dom_to_str( + obs["dom_object"], + extra_properties=obs["extra_element_properties"], + with_visible=True, + with_clickable=True, + with_center_coords=True, + with_bounding_box_coords=True, + with_som=True, + ) + assert 'box="(' in dom + assert 'center="(' in dom + assert 'clickable="1" som="1" type="submit" value="Submit" visible="1"' in dom + assert 'head bid="1" visible="0"' in dom + assert 'clickable="1" for="email" visible="1"' in dom + assert "Text within in non-html tag" in dom + assert "Text that should not be visible" in dom + + dom = flatten_dom_to_str( + obs["dom_object"], extra_properties=obs["extra_element_properties"], filter_som_only=True + ) + assert 'for="email"' not in dom + assert 'type="submit" value="Submit"' in dom + assert "Text within in non-html tag" not in dom + assert "Text that should not be visible" not in dom + + dom = flatten_dom_to_str( + obs["dom_object"], + extra_properties=obs["extra_element_properties"], + filter_visible_only=True, + ) + assert " tuple[str, dict]: + goal, info = super().setup(page) + + if self.remove_human_display: + get_utterance_func = "getUtterance_legacy" + else: + get_utterance_func = "getUtterance" + + # this task requires specific treatment to recover the text goal + page.evaluate( + f"core.{get_utterance_func} = function () " + + r"""{ + query_div = document.getElementById('query'); + if (query_div.children.length > 0) { + utterance = ''; + utterance = utterance + query_div.childNodes[0].textContent.replace(/\s+/g, ' ').trim(); + utterance = utterance + ' "' + query_div.children[0].getAttribute('class').split(' ')[1] + '"'; + utterance = utterance + ' ' + query_div.childNodes[2].textContent.replace(/\s+/g, ' ').trim(); + } + else { + utterance = query_div.textContent.replace(/\s+/g, ' ').trim(); + } + return utterance; +}; +''; +""" + ) + + # re-extract the goal + goal = self._get_goal() + + return goal, info + class ClickOptionTask(AbstractMiniwobTask): desc = "Click option boxes." @@ -610,8 +643,8 @@ class UseColorwheel2Task(AbstractMiniwobTask): desc = "Use a color wheel given specific random color." subdomain = "use-colorwheel-2" - def setup(self, seed: int, page: playwright.sync_api.Page) -> tuple[str, dict]: - goal, info = super().setup(seed, page) + def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]: + goal, info = super().setup(page) if self.remove_human_display: get_utterance_func = "getUtterance_legacy" diff --git a/miniwob/src/browsergym/miniwob/base.py b/miniwob/src/browsergym/miniwob/base.py index 1c394cf6..84bb8d3a 100644 --- a/miniwob/src/browsergym/miniwob/base.py +++ b/miniwob/src/browsergym/miniwob/base.py @@ -11,7 +11,8 @@ class AbstractMiniwobTask(AbstractBrowserTask): """ - nondeterministic = False + # gym metadata (default value, can be overloaded per task) + nondeterministic: bool = False @classmethod def get_task_id(cls): @@ -19,17 +20,26 @@ def get_task_id(cls): def __init__( self, + seed: int, base_url: Optional[str] = None, episode_max_time: int = 1000000, remove_human_display: bool = True, ) -> None: """ Args: + seed: random seed. base_url: str (optional), the base Miniwob URL where the task's HTML file is to be found. If not provided, the MINIWOB_URL environment variable will be used. episode_max_time: int, episode max time in milliseconds. Default: 1000000 ms. remove_human_display: bool, whether or not to remove the human display (goal, time left, last reward etc.) from the DOM. Default: True. """ + super().__init__(seed) + + # task properties, will be used to set up the browsergym environment + self.viewport = {"width": 500, "height": 320} + self.slow_mo = 100 # ms + self.timeout = 5000 # ms + assert episode_max_time > 0 # if not provided, try to get Miniwob URL from environment variable @@ -45,7 +55,7 @@ def __init__( self.episode_max_time = episode_max_time self.remove_human_display = remove_human_display - def setup(self, seed: int, page: playwright.sync_api.Page) -> tuple[str, dict]: + def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]: self.page = page # navigate to the task's url @@ -118,7 +128,7 @@ def setup(self, seed: int, page: playwright.sync_api.Page) -> tuple[str, dict]: core.EPISODE_MAX_TIME = {episode_max_time}; core.startEpisodeReal(); """.format( - seed=seed, + seed=self.random.randint(0, 1000000), episode_max_time=self.episode_max_time, ) ) diff --git a/miniwob/tests/test_base.py b/miniwob/tests/test_base.py index baeacf90..9fb3ba1f 100644 --- a/miniwob/tests/test_base.py +++ b/miniwob/tests/test_base.py @@ -30,8 +30,8 @@ def test_validate_teardown(task_cls): context = browser.new_context() page = context.new_page() - task = task_cls() - task.setup(seed=42, page=page) + task = task_cls(seed=42) + task.setup(page=page) reward, done, msg, info = task.validate(page, []) @@ -51,8 +51,8 @@ def test_episode_max_time(task_cls): context = browser.new_context() page = context.new_page() - task = task_cls(episode_max_time=0.2) - task.setup(seed=42, page=page) + task = task_cls(seed=42, episode_max_time=0.2) + task.setup(page=page) time.sleep(0.5) @@ -78,8 +78,8 @@ def test_remove_human_display(task_cls): context = browser.new_context() page = context.new_page() - task = task_cls(remove_human_display=True) - task.setup(seed=42, page=page) + task = task_cls(seed=42, remove_human_display=True) + task.setup(page=page) for element_id in ["reward-display", "click-canvas", "sync-task-cover"]: element_in_dom = page.evaluate(f"!!document.getElementById('{element_id}')") @@ -100,8 +100,8 @@ def test_remove_human_display(task_cls): context = browser.new_context() page = context.new_page() - task = task_cls(remove_human_display=False) - task.setup(seed=42, page=page) + task = task_cls(seed=42, remove_human_display=False) + task.setup(page=page) for element_id in ["reward-display", "click-canvas", "sync-task-cover"]: element_in_dom = page.evaluate(f"!!document.getElementById('{element_id}')") diff --git a/miniwob/tests/test_click-menu-2.py b/miniwob/tests/test_click-menu-2.py new file mode 100644 index 00000000..3851da2f --- /dev/null +++ b/miniwob/tests/test_click-menu-2.py @@ -0,0 +1,84 @@ +import os +import gymnasium as gym +import re +import pytest + +# register gym environments +import browsergym.miniwob + +# bugfix: use same playwright instance in browsergym and pytest +from utils import setup_playwright + +__SLOW_MO = 1000 if "DISPLAY_BROWSER" in os.environ else None +__HEADLESS = False if "DISPLAY_BROWSER" in os.environ else True + + +@pytest.mark.parametrize("seed", range(5)) +def test_cheat(seed): + env = gym.make( + "browsergym/miniwob.click-menu-2", + headless=__HEADLESS, + slow_mo=__SLOW_MO, + action_mapping=None, + ) + obs, info = env.reset(seed=seed) + + assert obs["last_action_error"] == "" + + match1 = re.match( + 'Click the "Menu" button, and then find and click on the item labeled "(.+)".', obs["goal"] + ) + match2 = re.match( + 'Click the "Menu" button, and then find and click on the item with the "(.+)" icon.', + obs["goal"], + ) + + assert match1 or match2 + + if match1: + item_label = match1.groups()[0] + item_classname = { + "Save": "ui-icon-disk", + "Prev": "ui-icon-seek-start", + "Stop": "ui-icon-stop", + "Play": "ui-icon-play", + "Next": "ui-icon-seek-end", + "Zoom In": "ui-icon-zoomin", + "Zoom Out": "ui-icon-zoomout", + }[item_label] + else: + item_classname = match2.groups()[0] + + action = f"""\ +page.get_by_text("Menu").click() +""" + + obs, reward, term, trunc, info = env.step(action) + + assert obs["last_action_error"] == "" + assert reward == 0 + assert term == False + + if item_classname in ("ui-icon-seek-start", "ui-icon-stop", "ui-icon-play", "ui-icon-seek-end"): + + action = f"""\ +page.get_by_text("Playback").click() +""" + + obs, reward, term, trunc, info = env.step(action) + + assert obs["last_action_error"] == "" + assert reward == 0 + assert term == False + + action = f"""\ +page.locator(".{item_classname}").click() +""" + + obs, reward, term, trunc, info = env.step(action) + + assert obs["last_action_error"] == "" + assert reward == 1 + assert term == True + + env.close() diff --git a/webarena/src/browsergym/webarena/__init__.py b/webarena/src/browsergym/webarena/__init__.py index ec79724e..22fc11ca 100644 --- a/webarena/src/browsergym/webarena/__init__.py +++ b/webarena/src/browsergym/webarena/__init__.py @@ -14,6 +14,6 @@ register_task( gym_id, GenericWebArenaTask, - kwargs={"task_id": task_id, "viewport": {"width": 1280, "height": 720}, "timeout": 10000}, + kwargs={"task_kwargs": {"task_id": task_id}}, ) ALL_WEBARENA_TASK_IDS.append(gym_id) diff --git a/webarena/src/browsergym/webarena/instance.py b/webarena/src/browsergym/webarena/instance.py index 80f8ba10..70648bf9 100644 --- a/webarena/src/browsergym/webarena/instance.py +++ b/webarena/src/browsergym/webarena/instance.py @@ -99,3 +99,12 @@ def ui_login(self, site: str, page: playwright.sync_api.Page): page.get_by_label("Username").fill(username) page.get_by_label("Password").fill(password) page.get_by_role("button", name="Sign in").click() + + case "wikipedia": + page.goto(url) + + case "map": + page.goto(url) + + case _: + raise ValueError diff --git a/webarena/src/browsergym/webarena/task.py b/webarena/src/browsergym/webarena/task.py index 2d4e4338..a7683c8e 100644 --- a/webarena/src/browsergym/webarena/task.py +++ b/webarena/src/browsergym/webarena/task.py @@ -20,11 +20,19 @@ class GenericWebArenaTask(AbstractBrowserTask): def __init__( self, + seed: int, task_id: Optional[int] = None, intent_template_id: Optional[int] = None, with_na_hint: bool = False, with_homepage_hint: bool = False, ) -> None: + super().__init__(seed) + + # task properties, will be used to set up the browsergym environment + self.viewport = {"width": 1280, "height": 720} + self.slow_mo = 1000 # ms + self.timeout = 10000 # ms + self.webarena_instance = WebArenaInstance() self.config_file: str = None self.with_na_hint = with_na_hint @@ -74,12 +82,10 @@ def __init__( self.task_configs = task_configs - def setup(self, seed: int, page: playwright.sync_api.Page) -> tuple[str, dict]: + def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]: # import webarena on instanciation from webarena.evaluation_harness.evaluators import evaluator_router - self.random = np.random.RandomState(seed) - # pick a task at random self.config = self.random.choice(self.task_configs) From 680e499dd4996abe7982d4aefc4986b40c03909d Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:33:14 -0400 Subject: [PATCH 03/13] ci tests code format --- .github/workflows/code_format.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/code_format.yml diff --git a/.github/workflows/code_format.yml b/.github/workflows/code_format.yml new file mode 100644 index 00000000..d3d10e98 --- /dev/null +++ b/.github/workflows/code_format.yml @@ -0,0 +1,31 @@ +name: Code Format + +on: + push: + branches: + - main + pull_request: + +jobs: + + build: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' # caching pip dependencies + + - name: Pip install + run: pip install black[jupyter]==24.2.0 blacken-docs + + - name: Code Formatting + run: black . --check \ No newline at end of file From 13b5046d6d592b11228e5ab06041b9a96d535612 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:35:08 -0400 Subject: [PATCH 04/13] pre-commit --- .pre-commit-config.yaml | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..2540c440 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +fail_fast: false + +default_language_version: + python: python3 + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: trailing-whitespace + exclude: ^(.*)\.md$ + - id: end-of-file-fixer + - id: check-yaml + exclude: ^(.circleci/recipe|recipe) # conda build recipes are templated + - id: check-added-large-files + - repo: https://github.com/pocc/pre-commit-hooks + rev: v1.1.1 + hooks: + - id: clang-format + args: [--style=file, -i] + - id: clang-tidy + args: [--fix, --fix-errors] + - repo: https://github.com/psf/black + rev: 24.2.0 + hooks: + - id: black + args: [--config=./pyproject.toml] + - repo: https://github.com/asottile/blacken-docs + rev: v1.12.1 + hooks: + - id: blacken-docs + args: [ '--line-length', '100' ] + additional_dependencies: [black] + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.5 + hooks: + - id: forbid-crlf + - id: remove-crlf + # Black does not clear tabs in docstrings + - id: forbid-tabs + files: '.*\.py$' + - id: remove-tabs + files: '.*\.py$' + args: [ '--whitespaces-count', '4' ] \ No newline at end of file From fac7a7d25eb007782e5886f7db9531d0281c2ff1 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:49:05 -0400 Subject: [PATCH 05/13] dev environment + unit tests --- .github/workflows/unit_tests.yml | 194 +++++++++++++++++++++++++++++++ dev/environment.yaml | 13 +++ dev/requirements.txt | 10 ++ 3 files changed, 217 insertions(+) create mode 100644 .github/workflows/unit_tests.yml create mode 100644 dev/environment.yaml create mode 100644 dev/requirements.txt diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 00000000..30ce018f --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,194 @@ +name: Unit tests + +on: + push: + branches: + - main + pull_request: + +jobs: + browsergym-core: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' # caching pip dependencies + + - name: Pip install + working-directory: ./dev + run: pip install -r requirements.txt + + - name: Pip list + run: pip list + + - name: Install Playwright + run: playwright install --with-deps + + - name: Run browsergym-core Unit Tests + run: pytest -n 1 --durations=10 -m 'not pricy' -v browsergym/core/tests + + browsergym-miniwob: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' # caching pip dependencies + + - name: Pip install + working-directory: ./dev + run: pip install -r requirements.txt + + - name: Pip list + run: pip list + + - name: Install Playwright + run: playwright install --with-deps + + - name: Fetch MiniWob + uses: actions/checkout@v4 + with: + repository: "Farama-Foundation/miniwob-plusplus" + ref: "7fd85d71a4b60325c6585396ec4f48377d049838" + path: "miniwob-plusplus" + + - name: Serve MiniWob + uses: Eun/http-server-action@v1 + with: + directory: "${{ github.workspace }}/miniwob-plusplus/miniwob/html" + port: 8080 + + - name: Run browsergym-miniwob Unit Tests + env: + MINIWOB_URL: "http://localhost:8080/miniwob/" + run: pytest -n 5 --durations=10 -m 'not pricy' -v browsergym/miniwob/tests + + browsergym-webarena-fast: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' # caching pip dependencies + + - name: Pip install + working-directory: ./dev + run: pip install -r requirements.txt + + - name: Pip list + run: pip list + + - name: Install Playwright + run: playwright install --with-deps + + - name: Run browsergym-webarena not slow Unit Tests + env: + SHOPPING: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7770/" + SHOPPING_ADMIN: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7780/admin" + REDDIT: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:9999" + GITLAB: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8023" + WIKIPEDIA: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing" + MAP: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:3000" + HOMEPAGE: "PASS:4399" + WEBARENA_PATH: "${{ github.workspace }}/webarena/" + run: pytest -n 5 --durations=10 -m 'not slow and not pricy' --slowmo 1000 -v browsergym/webarena/tests + + browsergym-webarena-slow: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' # caching pip dependencies + + - name: Pip install + working-directory: ./dev + run: pip install -r requirements.txt + + - name: Pip list + run: pip list + + - name: Install Playwright + run: playwright install --with-deps + + - name: Run browsergym-webarena slow Unit Tests + env: + SHOPPING: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7770/" + SHOPPING_ADMIN: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7780/admin" + REDDIT: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:9999" + GITLAB: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8023" + WIKIPEDIA: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing" + MAP: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:3000" + HOMEPAGE: "PASS:4399" + WEBARENA_PATH: "${{ github.workspace }}/webarena/" + run: pytest -n 5 --durations=10 -m 'slow and not pricy' --slowmo 1000 -v browsergym/webarena/tests + + ui_copilot: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' # caching pip dependencies + + - name: Pip install + working-directory: ./dev + run: pip install -r requirements.txt + + - name: Pip list + run: pip list + + - name: Install Playwright + run: playwright install --with-deps + + - name: Run ui_copilot Unit Tests + run: pytest -n 5 --durations=10 -m 'not pricy' -v ui_copilot \ No newline at end of file diff --git a/dev/environment.yaml b/dev/environment.yaml new file mode 100644 index 00000000..698128e2 --- /dev/null +++ b/dev/environment.yaml @@ -0,0 +1,13 @@ +name: browsergym-dev + +channels: + - huggingface + - conda-forge + - defaults + +dependencies: + - python>=3.10 + - pip + + - pip: + - -r requirements.txt \ No newline at end of file diff --git a/dev/requirements.txt b/dev/requirements.txt new file mode 100644 index 00000000..749af812 --- /dev/null +++ b/dev/requirements.txt @@ -0,0 +1,10 @@ +black[jupyter]==24.2.0 +blacken-docs +pre-commit +pytest==7.3.2 +pytest-xdist +pytest-playwright +tenacity +-e ../core # local package +-e ../miniwob # local package +-e ../webarena # local package From 4b727941602e4d052507bbb24199de37e1b78628 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:49:31 -0400 Subject: [PATCH 06/13] black formatting --- demo_agent/agents/dynamic_prompting.py | 4 +++- demo_agent/run_demo.py | 17 +++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/demo_agent/agents/dynamic_prompting.py b/demo_agent/agents/dynamic_prompting.py index 44d5f064..2714eb3e 100644 --- a/demo_agent/agents/dynamic_prompting.py +++ b/demo_agent/agents/dynamic_prompting.py @@ -37,7 +37,9 @@ class Flags: use_concrete_example: bool = True use_abstract_example: bool = False multi_actions: bool = False - action_space: Literal["python", "bid", "coord", "bid+coord", "bid+nav", "coord+nav", "bid+coord+nav"] = "bid" + action_space: Literal[ + "python", "bid", "coord", "bid+coord", "bid+nav", "coord+nav", "bid+coord+nav" + ] = "bid" is_strict: bool = False # This flag will be automatically disabled `if not chat_model_args.has_vision()` use_screenshot: bool = True diff --git a/demo_agent/run_demo.py b/demo_agent/run_demo.py index dadc647a..39892ee6 100644 --- a/demo_agent/run_demo.py +++ b/demo_agent/run_demo.py @@ -30,7 +30,10 @@ def parse_args(): "--slow_mo", type=int, default=500, help="Slow motion delay for the playwright actions." ) parser.add_argument( - "--headless", type=str2bool, default=False, help="Run the experiment in headless mode (hides the browser windows)." + "--headless", + type=str2bool, + default=False, + help="Run the experiment in headless mode (hides the browser windows).", ) parser.add_argument( "--demo_mode", @@ -82,16 +85,18 @@ def parse_args(): def main(): args = parse_args() - task_kwargs={ + task_kwargs = { "viewport": {"width": 1500, "height": 1280}, "slow_mo": args.slow_mo, } if args.task_name == "openended": - task_kwargs.update({ - "start_url": args.start_url, - "wait_for_user_message": True, - }) + task_kwargs.update( + { + "start_url": args.start_url, + "wait_for_user_message": True, + } + ) exp_args = ExpArgs( agent_args=GenericAgentArgs( From 96fcea7eb129a55fcf14b89f484393e3fd806053 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:51:30 -0400 Subject: [PATCH 07/13] rename workflow --- .github/workflows/pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 367e170b..3982f754 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,4 +1,4 @@ -name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI +name: Build and Publish on: [push, workflow_dispatch] From 935d08869db6ab7d601822510f8de723757f1017 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:52:57 -0400 Subject: [PATCH 08/13] unit test update --- .github/workflows/unit_tests.yml | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 30ce018f..a8e77293 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -161,34 +161,3 @@ jobs: HOMEPAGE: "PASS:4399" WEBARENA_PATH: "${{ github.workspace }}/webarena/" run: pytest -n 5 --durations=10 -m 'slow and not pricy' --slowmo 1000 -v browsergym/webarena/tests - - ui_copilot: - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - steps: - - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - cache: 'pip' # caching pip dependencies - - - name: Pip install - working-directory: ./dev - run: pip install -r requirements.txt - - - name: Pip list - run: pip list - - - name: Install Playwright - run: playwright install --with-deps - - - name: Run ui_copilot Unit Tests - run: pytest -n 5 --durations=10 -m 'not pricy' -v ui_copilot \ No newline at end of file From 6f6b5e158268fe809ef36bf5dfb3fa1ff7e3daf8 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:54:21 -0400 Subject: [PATCH 09/13] unit tests fix --- .github/workflows/unit_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a8e77293..f33484f8 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -36,7 +36,7 @@ jobs: run: playwright install --with-deps - name: Run browsergym-core Unit Tests - run: pytest -n 1 --durations=10 -m 'not pricy' -v browsergym/core/tests + run: pytest -n 1 --durations=10 -m 'not pricy' -v core/tests browsergym-miniwob: runs-on: ubuntu-latest @@ -82,7 +82,7 @@ jobs: - name: Run browsergym-miniwob Unit Tests env: MINIWOB_URL: "http://localhost:8080/miniwob/" - run: pytest -n 5 --durations=10 -m 'not pricy' -v browsergym/miniwob/tests + run: pytest -n 5 --durations=10 -m 'not pricy' -v miniwob/tests browsergym-webarena-fast: runs-on: ubuntu-latest @@ -121,7 +121,7 @@ jobs: MAP: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:3000" HOMEPAGE: "PASS:4399" WEBARENA_PATH: "${{ github.workspace }}/webarena/" - run: pytest -n 5 --durations=10 -m 'not slow and not pricy' --slowmo 1000 -v browsergym/webarena/tests + run: pytest -n 5 --durations=10 -m 'not slow and not pricy' --slowmo 1000 -v webarena/tests browsergym-webarena-slow: runs-on: ubuntu-latest @@ -160,4 +160,4 @@ jobs: MAP: "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:3000" HOMEPAGE: "PASS:4399" WEBARENA_PATH: "${{ github.workspace }}/webarena/" - run: pytest -n 5 --durations=10 -m 'slow and not pricy' --slowmo 1000 -v browsergym/webarena/tests + run: pytest -n 5 --durations=10 -m 'slow and not pricy' --slowmo 1000 -v webarena/tests From df42abaf708488ff2fa64d3d23858dba76074d28 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 16:59:47 -0400 Subject: [PATCH 10/13] workflow update --- .github/workflows/code_format.yml | 31 ------------------------------- .github/workflows/unit_tests.yml | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 31 deletions(-) delete mode 100644 .github/workflows/code_format.yml diff --git a/.github/workflows/code_format.yml b/.github/workflows/code_format.yml deleted file mode 100644 index d3d10e98..00000000 --- a/.github/workflows/code_format.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Code Format - -on: - push: - branches: - - main - pull_request: - -jobs: - - build: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - steps: - - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - cache: 'pip' # caching pip dependencies - - - name: Pip install - run: pip install black[jupyter]==24.2.0 blacken-docs - - - name: Code Formatting - run: black . --check \ No newline at end of file diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f33484f8..9183c77e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,6 +7,31 @@ on: pull_request: jobs: + + code-format: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' # caching pip dependencies + + - name: Pip install + run: pip install black[jupyter]==24.2.0 blacken-docs + + - name: Code Formatting + run: black . --check + browsergym-core: runs-on: ubuntu-latest From 1035aa63946eaefbd25262ce84e76331e778c27a Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 17:02:34 -0400 Subject: [PATCH 11/13] remove matplotlib import --- core/src/browsergym/core/spaces.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/browsergym/core/spaces.py b/core/src/browsergym/core/spaces.py index 338dc86d..177959e5 100644 --- a/core/src/browsergym/core/spaces.py +++ b/core/src/browsergym/core/spaces.py @@ -2,7 +2,6 @@ from typing import Any -from matplotlib.pylab import Generator import numpy as np from gymnasium.spaces import Space, Box, Text from gymnasium.spaces.utils import flatdim, flatten, flatten_space, unflatten From 806be24dcb1a21704f65e944059410fc40c669d6 Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 17:02:42 -0400 Subject: [PATCH 12/13] gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4a48538f..078c6d62 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -.DS_store \ No newline at end of file +.DS_store +__pycache__/ +*.py[cod] \ No newline at end of file From e4b7a3f63bb7eaa19d89d29537f34f315818b48a Mon Sep 17 00:00:00 2001 From: Maxime Gasse <maxime.gasse@gmail.com> Date: Thu, 9 May 2024 17:07:37 -0400 Subject: [PATCH 13/13] version bump --- core/src/browsergym/core/__init__.py | 2 +- miniwob/requirements.txt | 2 +- miniwob/src/browsergym/miniwob/__init__.py | 2 +- pyproject.toml | 10 +++++----- webarena/requirements.txt | 2 +- webarena/src/browsergym/webarena/__init__.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/browsergym/core/__init__.py b/core/src/browsergym/core/__init__.py index 31df8fe9..127356ae 100644 --- a/core/src/browsergym/core/__init__.py +++ b/core/src/browsergym/core/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.0rc7" +__version__ = "0.2.0" import playwright.sync_api diff --git a/miniwob/requirements.txt b/miniwob/requirements.txt index 11fd6522..984338fe 100644 --- a/miniwob/requirements.txt +++ b/miniwob/requirements.txt @@ -1 +1 @@ -browsergym-core==0.1.0rc7 +browsergym-core==0.2.0 diff --git a/miniwob/src/browsergym/miniwob/__init__.py b/miniwob/src/browsergym/miniwob/__init__.py index 2e66d458..9ca3a3f2 100644 --- a/miniwob/src/browsergym/miniwob/__init__.py +++ b/miniwob/src/browsergym/miniwob/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.0rc7" +__version__ = "0.2.0" from browsergym.core.registration import register_task diff --git a/pyproject.toml b/pyproject.toml index 917640fd..707a375b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,12 +25,12 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", "License :: OSI Approved :: Apache Software License", ] -version="0.1.0rc7" +version="0.2.0" dependencies = [ - "browsergym-core==0.1.0rc7", - "browsergym-miniwob==0.1.0rc7", - "browsergym-webarena==0.1.0rc7", - "browsergym-workarena==0.1.0rc7", + "browsergym-core==0.2.0", + "browsergym-miniwob==0.2.0", + "browsergym-webarena==0.2.0", + "browsergym-workarena==0.2.0", ] [tool.setuptools] diff --git a/webarena/requirements.txt b/webarena/requirements.txt index bbd6fb1f..200725f0 100644 --- a/webarena/requirements.txt +++ b/webarena/requirements.txt @@ -1,2 +1,2 @@ -browsergym-core==0.1.0rc7 +browsergym-core==0.2.0 libwebarena==0.0.2 diff --git a/webarena/src/browsergym/webarena/__init__.py b/webarena/src/browsergym/webarena/__init__.py index 22fc11ca..1c5d0753 100644 --- a/webarena/src/browsergym/webarena/__init__.py +++ b/webarena/src/browsergym/webarena/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.0rc7" +__version__ = "0.2.0" from browsergym.core.registration import register_task

|Il7D)tGbZR0;b{8%?XBa*Myd{33KD)jb`XWign$5vx zx(lZ{&>Zj$%c7l>CC}1g3*ru0D5iGPUAydJTuLNU*#Usn(UFiQ!5nk7x3fFJ=|WPv z=dWk1IeXbJ7Mp%;Zsz{zuqX#4lJTj;Dnrql@UaD}L?ZDx?&o8-2}+C@1TT*M%7o?F znuW~{tu*N&D(T4StWrX&n|cx)!j1m2QoJm#3L^mu$aX-GPK@zqD}84We| zsS+6A)U3I~na!)FiqmC{su}N;Owd$?asg47QjsrL4 zg^nXo-z=8Y%{-ATX-it$K6Id!-!KaAt9zY$OZdVR-uKYu<$x>;c65Eb1ZxZ?SS$d} zbb*(^KgUV@C|!Z~X{C&;)B`TJ7BYM0tGvGm_k%0OQCxj*!qHNZopw?9QHAe3i44Oa z?g=_g_b)RopTa{sy5YcB(G4m`yiiH_g~DJ!NN z3kEk0m3C=&WS+z=MhuG+H}hN$H^Z)s+{o@YIfe<3)1*Iwwjgn~lukFja&p0fnR4Tl zyT$sAk*|z=Z8ToKF=5#V;wo*kU#h%N`BHVfVpGDlDeWr%prj&IvL;coChp&xEa{G0 zyHz&FaNwkWk93%pGx1xhjLAsR?X25_xOH-rS*5AERvTr#KqbFM-m~NpJfvGSV`n4b zQ|k5VR**8Jkdf6pnCc^;!83hlL*YSOt1&Pz46*ts?(tWDuR@k+p)3AREfdg8_DJW= z&cM@;P~dk5&d2C>C`monRF=KOD}Xb?;gRSm#hy)5%Cq6XNK`sUJ2#|wa#L}M6ic+> z9<0HT9EL65bJRQIqlb>F`ebZ){Gx%(=x%mPHtxozUmcLun{FIxBtSQFvO^z=Au1~_ z^@j)|kpcB53Dj#BN~mQ=&Tn6+TImf)mi`lYd#QjebC_Da!DfbXM#&=pq^il8()#x) z`gQUcVY7|RB6);|8Q_p;A>~m)V4BsR7}z6)kc;u`*rtA3xWeV?m&pR4>8=Y5~^!2f%$4kh2` zZ2y(pbFawGx6Xh8C5H4Khsd286TfGvq>QhcJ$RSHW2qh7Yl^Q(@D(2!p5*x>IQ7RK zpqvbqOIp%%hMZD9e%BHQPb)#u1NdQ@Oe-7t&Ls|>mQT@s-pW_ZZkn^cw&yN~;8H~` zUpgC`JNVl3cR2*90atmtxb(x~PJUgwtYZ57<-lDd!gtQ{27c8?_5(a$@gYb4(qjg$ ryduuI?-W@OE9biS{e1kfCqCrL`BihqIpMnfE(Pxfto%Opwx9nEr;3+K diff --git a/demo_agent/utils/__pycache__/llm_utils.cpython-312.pyc b/demo_agent/utils/__pycache__/llm_utils.cpython-312.pyc deleted file mode 100644 index 8e6b94240af8b7fb5581531314b0665263af0008..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18449 zcmdse32+?OnO^ry&wb)PNPq(p1Ti28iV_d;5VR0UQIL39q$CTXG#a2Az<_hW-2+k> z4`f)Yib0FJ0%~^)a4f~6UAqj1Ohm`2m|NLoWyhP{)TSyk8rT@OsYKo=l~Pqw1ryxL zE}e?<{jZN1fRMBmyS9=viPvwsU;ppu|NZ~>AN`j;pPR!m{`N1W$RUpV&-BAMYsF1-z%H>%+>DkzIJe-uo3(01t1Z_p zdYcEtonlDZBeqFqDtHNv&f?Dt_wKjDzDaw|2~D?<61Y{+O;7jcot8ArT6ZtG>tlX3(YmwX221j#Q zK8=;oj;v+OWz@@O(Q;j;J3BA*pbS46Rz9Y#!#33-OR8OwqVmwF zYE496QEkKVR5Y#HMQc-X4?BwFT(PE@T)#F7zN0_6yeCqm;$m(0s{`2Zwr@9E=c2zKrZo94*WT@ zj;!S(uE=j?omo4p&AYfPH)e7P(bJABZ@kqhSC${EGOm};S(nj*=PX>-nstp;m+u-i zqs;}Z?v!`RH|5XzWJ}fy9&w1yj1N~GSwY?+`JEsC^ zlOxA!jlRm~Y{2N32QO|k`gDWQI_tmTGg=#GJY}qv-CQgF>U_G%cuMrMkp|wj8qd7O zZ{ujawWFPrC#T#M`^i0KH|~|sB2NUfHggwxD(1!XPPr}BL`&9#Rj`(O<+(K0P&>x_ zMy-6#dW`X5PR^`{t+j;R<#YNIvQG48%i2-e-?o)$y~aJyiH<4vtL|C;>MpHz znZL}vVtt;w%!h-~ZwnmvG;OpHq@6q&OO7xpK)f1Ckc?A8mt*Nsc2Sm;@l;aLB?AeI z@u-Y#7Me(6GlgPlH}+)++jc@uDxqlTQZybDshp5{-0WWeFe4OADwpwCObI1ZX{}i| z9!F2{I6Y4SB`JoY;}F+kD4i+~i8|Fr!h*e zV2n!ak}<=xQ8}8B(7keqwb@lZ*l$`Pwz!?c6Um`;ER_t04q>h0(c}oOLxLVP^;9Sw zy+|(vcOkz*sbNFtVsVqANv^!;Ya45;tn`=-_0>{BokWlNrRBVjwvq}BN9FQk`Z9(( zV@dNN$naQVA`wbXBnGk67@9)r5_}NV-e63s=vZfy@|0=D#{A2()W^r(y~ah_gwhk^ zamk#8$&SORINk&`8IDPDQD1Oc4LTUx=~HC=>zvqa64G0NMQJ#LDZF~T;8;?O4Mo#L zHl}f_MhWkq@$=f}8M7K{i_tMA0q)V*s)Bph6EwNs98)|s66&O(gb#rkCA=S*_>eBs zyVCZC@gB>sXajW_>SQVwbm&1eVDGD}-N9*!G?Yq;irG0Wh_-cfU9TaR9&KOJuEbE- zS~8ktS_~;rH%yDKp{I{k--1tTvSIDkFx#_**2Y`;#@KzCM0TceEePpmXoX%?rdq+) z98Ssl8sS<`=7IC6G}y6XoGCdZ1C@d20n4y=ha^dq#GcS|02z`P$_PrRHH%?Xlz~U`d%Y1cM%`5S2$1)d2<=!JJeF!CI;a8YEKU zk~FT`HAyLN!(*xtOJW@eTd3}5N%m?8HYjhVOHPA%RENH=RXZD&K?UaV?k`$zSy*)xEH%7{XX zP25*`$fR>s8*42;g8EkcDK7&AO>-+JdCupbK3-}8zG^PH1G786zB}K#H-F&F`>khI zEM8Y#sjg}Mg}E1Ujm5gIna5X}Iak%2fh&Op`;v3ZFFLvxpMCeGYcCZ$juf2RW^6MP zvx%R1f~A_qnUfzneX~_p+%vYHdHki?or}j858Q0KdE(~5TTlIk{Xh8soj-r#x%_j3 z`JwUr;Ft2%^32iO-kLX0UOBl?v*c|lIfK;ATi(SNml_W*IghM7#Cd|{;T2oY+}+Q4 z>hcYTZ=JYx@V)+g-RYwHOkOziH+T1N!PYyR+f{eFv1@V9yZf*0zu9)}V19ppvGK{7 ze(L4Sl{5L){Wm9XUd;EM%Wpov^Ml!Z_SJmtSMFLZRn;qYuB!e6Z|$#E>d^Z= zr4cJVTlLsJ?nnDv#|~J3)Z5o{>_O{~AGG1e5CDUD5W2JCZ*>vdv(k>qnpVd#?!~_X zdK4IfrMByiEI(pm*s)65dOHn!P1({Oo#h&GS;P)rgwdr3;Uvf1Ou80^WOyOB?12F zi1)K_DWO&vrxOx{GE^!cH-m-@29HQdiRr3j1oAl=k4td?Bf`x}2=ZOSv8Hw4H_>HGLF*(ieJnW?1CH#5Swk~V4NhRGu;@t*RT0JqwjT!~V51&N5#+uk z$)i$Kq(LzVMQ{S<5E~y`FwIoJ?Kl**#Ga(*KsR^o)RS#h7GV6ame-f!?06gG=-X($H#jYZS^M?b7vf#8XN;3R^V05 zdG!CLB~mts|39)o)|xe%bRe@xt1)ASLfh|QUJ8keO@vDAd3$@`wuAOEjUKKVojc;Fb&m$y(Y zL`54F1U2P$Dz>8lFt^FD&n4Aq>}1tbmPU$9Fi0lsqFOVtan+%ZN5kWmBms1J1VCEu zriSDKlzUKws~9GichiLjsMv!-rzx}OngN?NRJ^F!pVnGD1>TgU#~edkgbz3UZH#%O-JdOQHHkfKFs-RW)Hma(j9KI zt#^iBvAA82mg<}56LX2&7jsV*>vu2LA1>4%zSUQ#@0~eux1M`&U+M5;KlL5GS$}1C z_SyND=3ZK^+gYgF`4fNl`@W+qR?CC#g0pAFK5Jj_FWT?gxj@yMu`96!d(ppT####2 z&nk1C8C$v1*pX8TjXP(2rK-k-nz_x(Roe?y+h^>*V5t4;CqE4CSmYPmu32*vi+zhD zi_hP@_+H1K_x!LY-}i<57cS&q9Lrz0nD@tLthb%snR8cNI#6f!fC1Wli;1PiBR_FI zvT_g;y?f+SaQXo*xaAJ#bUg~(9ITq{`}(C)eZ#DyR69(zKybF^YJaI=OU|0Bo}HR` z;!_vucTfCkr3zI3=bd`ae~eeiKAYWgjI-R@c0AzzG3Pq&wf)%j@NtLj&m1nHft?ya z!}RMK6v#iCMRX#|6OL^*>A`~xgd65Fy)cmxSh$#`Ukn4eO$vuhH}SShL6Mff&EDQt(K4BD!j%vg<*YE#lt znJ$rgGm?b6PPO7$)vn!FXlb=gqV0}AS|6xRKz;!YzJovI3n&m1a91z8w-?;ob7MvK zp6R0{U+~R=D+3Ec(bqEl_-%)KX8&)0dG?utqj_QAeaDVcp!&_smCQm%u5NMr&8|Y% zk$iJ+G4N>K`RIp%ni-G02~>ylbw`!s=zXzw{T3?opArcWO)E;?k*3z{*vO&+#x~!u z(T;r9#=91+WYZYG&T(ZsV{8LocbRkWmW9)*Jq3qoG{z+A#}Klg;2pTxiYp|E<+GqI z)576BjK^bXnXEQ7KzbwsDUCo>41hGu%U{HGg_a?ty$0Wi1b>1NX_E(W=Ur;@7>a4` zLr2xJV{^f=dEtvoj*i=bhUGw4A<(sWXnE)1!p_6DjuvPDbTo4t1^1uwh7fy21}18NgOG0oj}H!RqS?$+s}w>|zDg|SzA zF|eiR3FU>*{ZlIkp0K_w@MOtyFw}3tl(m+Fky$I4LSGWuo3(BvTF~B{cGj9U4M6j` zVAfJ*AcUWI;|>dPSgsyk^SsGmY15b=E|1~KtQDh#2$~Wqo)Xe#K*gY~G?a~a(VDKo zDzT$12w_5-4F4JrLscNA7j0y&D*V-l){I&ZZFKj(u|FGFb&<_%W6KRA$v_~->;^0sxXdAv~ zb^+3TfRaYhbE%mVA9?C# zN8VDFJljg%hPRF^I&Rh#HytQ=4_tZbBVWUU|hsoSzkLH?-y2 zZwN~byKXnPmztkn8h9xm87>Zt#+44hlnw# zXvanx72d8*5J{$1%?gIJ=eZcXTJS|#L}3asSJ5^lyeeGn(avS>RM?*BK1;z=@avcY z>)BvvwqL;D4CtFNv!3%R_ju;@=Tj31Xc$ph@ID2Uj3@(<=?R@hP>XOEg;ZIN1_dp# zErz19G&~V!I1J>`4YDhGLdR0c9Uvi+4+Kw)rw|jEBndE-N+b}}=njo(ZWD5KkTa_1 zbXZUY>PxjDr2{re;K_4mP9J0Ni$xxoVRX&LSa+RG!pxiXjO-y|#eEL>B>@;km?t+?p^J%y4KUTX?*xtcz|_4`$QF6$3n zHdNQ(F;a4jxBd#1N@TDRXNmB8g1e|ajw#|qQ(QbV^hm-|Y_{wm;ZpUKn;^m_M!1Rd~!Qfy2oPXhRF}UM% z`S;DPWy{(&g1*-=5IPWMsTF|ODhX0ozI zjY@+uuprzh7cepk-DKUOh0If+H!lJ+mPnl=UH~WUQ~}Y7;P`BkLIk zPSk@IHUur~nc<^IOM$ef)EXU!z-=!}nF8aHtPm}^x(@&IK?&-&Xc^bFOPt8sJ-dFB$R$J{A z&x|l5l@Jx)^Txr2+T8Q+H}(AUhi<07fAE(4{oc~9Ju^=%xM#DuJ-MfI$MSXIg14*W z_0P6lIdSE&6$c1F&oykCkIlt$u3~-H^qEq9>wI!9S*-84!|}FW#ICI~&yB4#mVhwyk!ZVNUqcbv4_SdGs?z$RvTVj7>)A2F zLn!eZd88@?1)M5uls1Y=ZnR;uHWqb723?6xz*yk+I+&&#q*AN0<B8@!P+c@qZL7Ly)3SlfzfboM(tS@wX9}^Ku{X7? zY3}BTChg>Jp$T&4IpvR0VH&n=Ps!b=S>peX!(gSg3s~ zoqOWDr`|dBM`wy#_P)0rjkcmu>vaFjSFjUozOSA6>Y3U8C7~6+s@w9xHf-=UyJ?rc zo19h4&c=eXapCB~K6rMI=MLnXcK*cKO`B@hhk|ces4WP!`MT{(LPx2(em*=GUg$4Y zcT7L^p{IVKw%}>W1qz(04jt>RSZ=rsh;iZFc;fu2lSus=(W2R9E1fv(zNjd3F09L>WIp3VqzTzw;vKTy zW65zC__}_iKH*O!?U=?rvro;5KwTkc&u|2+r)0J!BMIurOp!NcLO#p2{)@_SvO4K0 zb9j9r`U!Ozg*BEySe(Yi@=-Ao&HZhTWnxf~(&PntndbR21*(ydO9}97CU`*anX+27 z6h5{-tg}9((X!}r4*3amr5d}O&I~06S5~Id5FwIJ#~)J$Dnl5Dt{#7?CtF7 zdNh1wx61d11r0SjfE`2xuT}f8Gw1tGo|JzZJODXC9!&WkqEO+jWI1qb^RpeVI>zA< zkz~LD){CUblx3ulbl11Nw(G0(m5z+2fvt%_eS6e@Q5<50nOUU>* zkBCui-R#ipGYg%$hjQCKnuhl+;{>=je zV#g|ZD(5+gqc4dDEchD5qPm$a&rAsTdR1qHg&86d)g7T|#Y9{}-5ZH~X(Ae@07N8m z4K4)GVQV1&W2#YsG&jxYc%3T=POSx9-x$k3jysj+Xcitz+M+jjQ?0@bB_ximYg0TG z3fg{MpAPv#p`I+6=lVk66h)8ZGjqKBN;Gzat6qU7LIwq8QjW8ze@x4Gp!USwx=%ux z7#ad_g~OqTzKlso$d!mBCgN$z$iR99*3oB^U&B$p6r+z!L*i!{o@!414i$%}IE>@qfpulma{2m9@H7~%7$&_VwIWvxs0HiryiFV+138oof8{r74Knujs z3iu&wyTr?z=z9ld*mxnMO+SJ`0S;`QeV<`jgJHHJGUp%%HnPr~T6Bl5kmRoLS(zDP zqM(~cy;-YL|0(_8dFUhintq-Y9UGyra_*VY>(z_MB{Okd)?_{yW}O(N>r?2M?O^QL zV7j@mfQ!qZC zM4vOV;-SwCDRUVga#V;I*0xfc_n9s$EhZqwU~5JvH7!WM0AK9EY+fh3h$=Z$*Hg|^ zor>X$G96P8GabnUdr4O_Qkq+|Y(;02LV!mjsgc9epx%fao1;P!my z!CTvI)#ZCn=R3~)BzX42>edCNP`z#DXeqQU*Y}Q{^m4)5f=}#bA9~}Jg|;`cKWo@t zs@=L=8!ps_OU=Evlv`5%=y1MquGt9ie6`$`K2aW};I zg0udjcS~+h!P}11@!H1u{<;2zWU+SF^eKq;g}R(C@7a|Xc4-nGGc^w~t>{<{p(yKm z(I#5(g{xJx;%iuh5K-Eo{}J$}k9dyaOCw1!%;f$x~8y*BD8~`zcDFq=m&IlWry4Hbi?TqS{zR?xU-&h>-;bGn;~I z8l^u)ZEegRkHhvzM3)aMks*P{-WxjmQ7w*q`?QasR)3 z>MwQ{jy#i(M3=e;ao3}N(6C(BQK;+4rN2A*&gAlrgM}Rj-`oFQXTGkZSoai~xl2tg z-`ezzP0LMPg{H2>z3(2pc5r#;BZZxhym#rn(R@=^vFXf=@H>8u#|^&5C>UEZ)@;X* zPs=hX=RTa{WDyPFP52>B*DTsfP%4{3RPmJ_ArARH6zit1E$JLhpPcZRexQb>EOZtF z+w#tBzrmsXh=%ZIDAtUL-r|}u8SnG=*N*AcE3Yn$76SzR*qF2l+hy8!@(n8f2^If_ ziub4>!ty_*qKxHl(N8L7hb>= zijh#hFLWA>lbMn*4M}^*Q5t@L4ZobsquMC@I*r{zvy=av3JJy9-O^ytj(~E6k@^^|TvuMTjU;SEAdgbENYJc1ME(*dSK*#- z-3|OKr(;cVX)^*bwt;jcZlr^KJW-?7E6Xgx&o@lN0`I!cxI0#F)XHZ#Qfc3=JB&WA zW3A{IBmQ1@7;O+vL?R&Z5Wu-Tv-haxatKA~W6^OSZ&{A&-i|@|9`^4IX^DGCvZt36 znvh{6T2$fb6OZ+&mXm#|6*X0O^6c>wB)+vsb*2Fx$mk*V#O|?iY2>g*s~bTEQeyIP z6!^qVWWC9>z^aSoZa7A!37jf`z% zJiTi#k~+Oww%>03x4gg}(_|nC!pFn>J-#QrPp0|FX%x&6G#ZZ$GOepcmAdG zn4b6&6|Yc1!C}ph{w=DJM@w}bH@qgA!-xD53~4iUp+%iZ_Q^#mh*Q?-mV3@4nlq_a z{%^QTdsF!Y1$@#x|8uVT=bYyQuJPxbi&dI_&iVffw~1X3uq!+MEBEjR+}01cCRTQ^ zmc9?Tx(~RvRlAMfLSIe#kXlh{Y5!&5D9@keOYYiLE32)Xv>d{Ph2~X`&Z`;T!PlVPZ{*t32WaSSmT30xnZYryETJd`Ly}1X!d+41*H*G)g Qe9u$ZcK9xbYm9~eH&(hNegFUf diff --git a/demo_agent/utils/__pycache__/prompt_templates.cpython-311.pyc b/demo_agent/utils/__pycache__/prompt_templates.cpython-311.pyc deleted file mode 100644 index 30a71b727d002cdc5e8775284b92d50a95e84d8b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4235 zcmb^!?{5^xb@s>ozO#*w_*d*?kxau8Y?Gua$Xu@&n>b(_C_d8aTC`nm$7W%7&zaq$ z@%1bn5mJrB2RBM&qz1`fLLK2Ie9WJ5hfdlipQv2v zG<+GK;m`QlxS$7&U?#|+5W+zo%_3F2g;d{t9-#;Dy5&B69yB^A6H@)RP$tYH98v>n zPz|Z!oydI==D_QgOa#NDMnB^-Q8l)R5?%kMhy>?|^O|KRL??1evG4-6EM*xxvBg!( z#zvzXeJ^h)W}_FHI^#8gJfteNlGPQ<%6eO|Ph;a<9lDzcqYS5_41WoPkP)zmeJY;> z3~LF^mPlKwFXLz9e%1?MI1(0@L^X6v%mmdi4&lHc+k^rS7Xe&Yjl#iVPS*vJGgj<0 zHdb`S#)YrK5)^toi7_fI<%qPx=16czHpvp7!) z>=bNZf6`9z1~sS}Z@==qSL`_dUy&-@6WV3+5tk5^{U9QCuT%@)%@ISfseUbMH?S#r z7b#rQIZ#CMKmcs8j=G=cFr4V_`cRt1Zi2 zWGd6KqGc189&E#${CGKf{M+8)^2pi0rysuY#pRvL&~$61cdptySB}o@0=viwdIz-HToH8^ z%oTkJjX!rdBbB>_irfrp4`%HKWOdsuj!5O(u^(}&u+?-~l#fPPM@t+8&fv?|U*n7X zZFH@3eJo6dpuk$jdJt`|9$Ab@}w>!1x%ZmFU4jW|LT{0HsUk`B?J z`@0tFz3iE9=W^0dwPUaemt}A(F2`v5nUc!<9=-5 zcpq(Y9J({4B9Iz)F2R%iuo9tn54OgSA9Xmxvok^!{8=cuj7|LE3P}~-?l6ZY*RscR zQqN@--Aau&X(nL_>j;2auDHTU%3>B$npEMPA1|*a(J$$WZvglY zNNaG~tq*^Fcl7bl*rTDbZKX1Fv^sROiDI&Xgiug)o%92pBVNmCCh3MIuu2&uBXrpB z-Vx*W50b>Vb$U9lu<-gzcs>eiS=Rvof8Xy%J#p_!KZI|k_hc1d_sMGa$vWafm$)a? z9W7RR)79Q|Ihy`vUwpIw72v*^>b{wEe=XRxG5X0wC3pbZ-LI_2 z*cs13WC_Z$sTf$6ov$Sb;s3COaV%fhe#7UguyvSLC=Q)~@POqQ8k zn6|UaB=qh`f`U;BsNyln5SkA$KqHj(8GusDQx|={Aoy(nT`1O5nk&uSoU28;N@q)F zZ=S6MBc<8W?9JI)-$3c?hENs-qb-#PSBi+8f?fu)=tBJ`v zg7()I`}`nq8K*qIp!@WMyEgzpatW|ewpzVPW3a=!&!Ee(--E|q)NjrdQF!BI}ZYo!5{(38GjJnlPuGoQq{gx9r3=j(i&)DZh4cs{p{QmGELUyjT_&*&P*zl12A_ zp_mH)i6vBP$s+tEK({cEEaoA1*#k3@EV`+TV%mG4V{Pb!$TBWBx1wp4>kN1&1W6gm zf`EtOuB}gA}PR9t+f6Dpe}G6S3~{f_WBLFP<~mh zp#$aiS{DUQtVaoqL?88eE?FEg7owgY>AHMEc zIB(Vx0wR$FDYg{FNV$`ExCbLR!6QE)Z+Tf`5>_Klij+u6-WH6M94|?A&&*;Ql1Qht zwKdhK zR-_oQqQxj*m&}+IFUAFwKsYX<<4BjUBRz6g^rhVmgccKe^g1dgMT8T2OpogcJ$Wy6 zSN6AyDGYyl`ZKYZp3mf3{?3+jf+vp|)Xm9W>ZnHXIHpva!Cq!+p1Rlyr|A#NmS%^u z#PEp#1@?rlxmwB8C@lqQ@K10kTGwH^j4&z+Ix321;lf1;%Q&KodD!8Y)Wl?l>5*cT zuSfYThT$q$o|g5*b-5VVlQ@B6I~bk6nu68jd^(r$S|*8O&AA0^&6%2uEBBKM40bJr zE2vC6M496>3S5p8(f%Z-8pYFwjj5t3)YWWVBf4VPE+*4j3BQr=T?N~;ObmNQaW7~t z06edh9NX0ln@fZRh2T%h2En>AZD3QUTo79yDO1<6#>U1+fc3oN_Hzy|O1ag&$OUF@ zPt*(#c}ZVm72CQOvjYD_33kiGE(H+))>|tqQa8*H`ZecY_nAHlM|#pV4nREB(5J|HD}=q0~)hwS}v%zTfbyY$!n& z-R&V>ZQlI1NS8jBLOWa)ao?$XW2b&VuO31Z$Lx$!Cl>!7(xaB&^c^;>5oQw4h;?D3u*&>m;#OH^M@Hs+B z8#y(D;w_9n)XXwIMu-FaC#G4@hm8Chia{0IaTQvgn{x0UIo-4bR%(x%K zK1ZcxGe<7vB@JrWEDcMTM8N1I1;C5?x_U8|WY~5Ii+94pim;d^8Ag!{$U! zrr4Y&EG8&R14ViHqk*#&6FOi))iNxnzoeB)hW4ES9bXzaN}S8Ed3@d)C>tsxpzZ)A zr46&PYnXH(NXu7+WNzMT@pV)Kw^VvJSET;U+ucm1Ujn#-UUZ{$$KuRq3y)L#>)Fob z!%K&sWOuJ-cds13KXGTGmfgLU&DYZTrHkU$+b<;a6fwoIxe=93iZdOCde}2qSpUatrOIB`SU=v7v*&`O z-6~G`rr8Ub;HwP9#MjtwBuueVv;2dZnYF(QzjfA8Pay%kS4HQxfklM>-X?w{EDHj< z*&(BO^rduJ$O)%&A^}E3So47F68jH$f)!*IVIlo8k)Odf{&bEERSs@RfTqT3pl{xE zN}5TB`kOq1AcA%R_#67?74)e6$b((K9=xS}@%EGK{?+y)UsHDQ)$Sgl^1WfxRIrIH zs0I|;H7`fpxEAzRRuI8wtO)QoKu~oEYZlik98&J~9_GH3d7o?Yg zT>!kOuPZ;8nU>$uYukF4$~*t#vIjsI{TKk``!>{;h1)#%^l|%<$E`=6w)fO}r`FmF zwRGW6Z)KM|mpUKr{o{_22Or()d(ypswR``H_J{5Rk9Lf#y)}9xT93CZ_AL)A4Xnm@ z!K`)rjm+22BdB{+pv=`@b`AH&PeUb*tE#P8SXI5Gs#=a-HW{8))lbTr8SKPVRd-6T zEI}-Jt*WZoP{K9WfZ$S9ZW8Wm-$E>($sPcA5b0q!^EhEGp95Bo=_G)kp=WX=5^uDi zOxv|n^;FBXll6G&+VOfv*Oilt(wf}Wkhev~gxePzh*bZfm4tW!uWd)!bkM z0sfG_U?2Z5cFt>o%}de+*nyG+&>yqS!V|8*dn7T)X0Gw^`s%g5$8s9qCCI z@w^wg#JVtLH*7{@b_TmaPN`u_BHss!qwu2)TtQDee(lk{D#K?}HCvQkM z&Vq9m_dQDW)U#c`7;8je{Y4a|cHWNOuG}_j2S2LqJhPUZs>xH_2bJ!8)qkc|hmt?# z9ep8Rg~usOD_!|&8Jg!Cc6#$wzn5yZ`-bhzrfo<;*7E)4&icBan4*_}R{<%3IfxhM zD46;*y!@k(#aLqBqBXx4Rtj-Q2V}m P+ummpsZU%*3RmFYjklh) From 48e5630494e6922a0ec05984861d6a835f2e2130 Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Thu, 9 May 2024 16:27:02 -0400 Subject: [PATCH 02/13] new release --- core/src/browsergym/core/action/__init__.py | 0 core/src/browsergym/core/action/functions.py | 146 +++-- core/src/browsergym/core/action/highlevel.py | 21 +- core/src/browsergym/core/action/python.py | 20 +- core/src/browsergym/core/action/utils.py | 122 +++-- core/src/browsergym/core/chat.py | 31 +- .../browsergym/core/chat_files/chatbox.html | 13 +- .../core/chat_files/chatbox_modern.html | 354 ++++++++++++ .../browsergym/core/chat_files/img/send.svg | 3 + core/src/browsergym/core/constants.py | 3 + core/src/browsergym/core/env.py | 179 +++++-- .../core/javascript/frame_mark_elements.js | 250 ++++++--- .../core/javascript/frame_unmark_elements.js | 3 +- core/src/browsergym/core/observation.py | 310 +++++++---- core/src/browsergym/core/registration.py | 14 +- core/src/browsergym/core/spaces.py | 40 ++ core/src/browsergym/core/task.py | 23 +- core/src/browsergym/utils/obs.py | 504 +++++++++++++----- core/tests/data/test_page_2.html | 63 +++ core/tests/test_actions_highlevel.py | 51 +- core/tests/test_gym_envs.py | 36 +- core/tests/test_observation.py | 174 +++++- miniwob/src/browsergym/miniwob/__init__.py | 2 +- miniwob/src/browsergym/miniwob/all.py | 37 +- miniwob/src/browsergym/miniwob/base.py | 16 +- miniwob/tests/test_base.py | 16 +- miniwob/tests/test_click-menu-2.py | 84 +++ webarena/src/browsergym/webarena/__init__.py | 2 +- webarena/src/browsergym/webarena/instance.py | 9 + webarena/src/browsergym/webarena/task.py | 12 +- 30 files changed, 1954 insertions(+), 584 deletions(-) create mode 100644 core/src/browsergym/core/action/__init__.py create mode 100644 core/src/browsergym/core/chat_files/chatbox_modern.html create mode 100644 core/src/browsergym/core/chat_files/img/send.svg create mode 100644 core/tests/data/test_page_2.html create mode 100644 miniwob/tests/test_click-menu-2.py diff --git a/core/src/browsergym/core/action/__init__.py b/core/src/browsergym/core/action/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/src/browsergym/core/action/functions.py b/core/src/browsergym/core/action/functions.py index 6b55f0ea..9eb06048 100644 --- a/core/src/browsergym/core/action/functions.py +++ b/core/src/browsergym/core/action/functions.py @@ -5,7 +5,6 @@ from .utils import ( add_demo_mode_effects, - check_for_overlay, get_elem_by_bid, highlight_by_box, smooth_move_visual_cursor_to, @@ -13,7 +12,7 @@ page: playwright.sync_api.Page = None send_message_to_user: callable = None -demo_mode: Literal["off", "default", "only_visible_elements"] = "off" +demo_mode: Literal["off", "default", "all_blue", "only_visible_elements"] = None """IMPORTANT The following primitives are meant to be included in the browsergym action using @@ -51,16 +50,16 @@ def fill(bid: str, value: str): Examples: fill('237', 'example value') fill('45', "multi-line\\nexample") - fill('32-12', "example with \\"quotes\\"") + fill('a12', "example with \\"quotes\\"") """ - elem = get_elem_by_bid(page, bid, demo_mode) + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=False) if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=False) elem.clear() delay = max(2000 / len(value), 10) elem.type(value, delay=delay) else: - elem.fill(value) + elem.fill(value, timeout=500) # https://playwright.dev/python/docs/api/class-locator#locator-check @@ -71,10 +70,9 @@ def check(bid: str): Examples: check('55') """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=True) - elem.check() + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=True) + elem.check(timeout=500) # https://playwright.dev/python/docs/api/class-locator#locator-uncheck @@ -83,12 +81,11 @@ def uncheck(bid: str): Ensure a checkbox or radio element is unchecked. Examples: - uncheck('65-5289') + uncheck('a5289') """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=True) - elem.uncheck() + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=True) + elem.uncheck(timeout=500) # https://playwright.dev/docs/input#select-options @@ -101,10 +98,9 @@ def select_option(bid: str, options: str | list[str]): select_option('48', "blue") select_option('48', ["red", "green", "blue"]) """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=False) - elem.select_option(options) + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=False) + elem.select_option(options, timeout=500) # https://playwright.dev/python/docs/api/class-locator#locator-click @@ -118,13 +114,11 @@ def click( Examples: click('51') - click('69-2', button="right") + click('b22', button="right") click('48', button="middle", modifiers=["Shift"]) """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=True) - + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=True) elem.click(button=button, modifiers=modifiers) @@ -139,13 +133,12 @@ def dblclick( Examples: dblclick('12') - dblclick('289-5-42', button="right") + dblclick('ca42', button="right") dblclick('178', button="middle", modifiers=["Shift"]) """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=True) - elem.dblclick(button=button, modifiers=modifiers, force=True) + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=True) + elem.dblclick(button=button, modifiers=modifiers, timeout=500) # https://playwright.dev/python/docs/api/class-locator#locator-hover @@ -154,15 +147,15 @@ def hover(bid: str): Hover over an element. Examples: - hover('12-8') + hover('b8') """ - elem = get_elem_by_bid(page, bid, demo_mode) + elem = get_elem_by_bid(page, bid, demo_mode != "off") if demo_mode != "off": box = elem.bounding_box() if box: center_x, center_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2 smooth_move_visual_cursor_to(page, center_x, center_y) - elem.hover() + elem.hover(timeout=500) # https://playwright.dev/python/docs/input#keys-and-shortcuts @@ -179,13 +172,12 @@ def press(bid: str, key_comb: str): Examples: press('88', 'Backspace') - press('48-6', 'Control+a') - press('48-6', 'Meta+Shift+t') + press('a26', 'Control+a') + press('a61', 'Meta+Shift+t') """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=False) - elem.press(key_comb) + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=False) + elem.press(key_comb, timeout=500) # https://playwright.dev/python/docs/api/class-locator#locator-focus @@ -194,12 +186,11 @@ def focus(bid: str): Focus the matching element. Examples: - focus('87-455') + focus('b455') """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=False) - elem.focus() + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=False) + elem.focus(timeout=500) # https://playwright.dev/python/docs/api/class-locator#locator-clear @@ -210,10 +201,9 @@ def clear(bid: str): Examples: clear('996') """ - elem = get_elem_by_bid(page, bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, elem, bid, demo_mode_type=demo_mode, move_cursor=False) - elem.clear() + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=False) + elem.clear(timeout=500) # https://playwright.dev/python/docs/input#drag-and-drop @@ -226,23 +216,21 @@ def drag_and_drop(from_bid: str, to_bid: str): Examples: drag_and_drop('56', '498') """ - from_elem = get_elem_by_bid(page, from_bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, from_elem, from_bid, move_cursor=True) - from_elem.hover() + from_elem = get_elem_by_bid(page, from_bid, demo_mode != "off") + add_demo_mode_effects(page, from_elem, from_bid, demo_mode=demo_mode, move_cursor=True) + from_elem.hover(timeout=500) page.mouse.down() - to_elem = get_elem_by_bid(page, to_bid, demo_mode) - if demo_mode != "off": - add_demo_mode_effects(page, to_elem, to_bid, move_cursor=True) - to_elem.hover() + to_elem = get_elem_by_bid(page, to_bid, demo_mode != "off") + add_demo_mode_effects(page, to_elem, to_bid, demo_mode=demo_mode, move_cursor=True) + to_elem.hover(timeout=500) page.mouse.up() # https://playwright.dev/python/docs/api/class-mouse#mouse-wheel def scroll(delta_x: float, delta_y: float): """ - Scroll horizontally and vertically. Amounts in pixels. Dispatches a wheel event. + Scroll horizontally and vertically. Amounts in pixels, positive for right or down scrolling, negative for left or up scrolling. Dispatches a wheel event. Examples: scroll(0, 200) @@ -370,6 +358,7 @@ def keyboard_press(key: str): keyboard_press('Backspace') keyboard_press('Control+a') keyboard_press('Meta+Shift+t') + page.keyboard.press("PageDown") """ page.keyboard.press(key) @@ -521,3 +510,48 @@ def tab_focus(index: int): page = page.context.pages[index] # trigger the callback that sets this page as active in browsergym page.locate("html").dispatch_event("pageshow") + + +# https://playwright.dev/python/docs/input#upload-files +def upload_file(bid: str, file: str | list[str]): + """ + Click an element and wait for a "filechooser" event, then select one + or multiple input files for upload. Relative file paths are resolved + relative to the current working directory. An empty list clears the + selected files. + + Examples: + upload_file("572", "my_receipt.pdf") + upload_file("63", ["/home/bob/Documents/image.jpg", "/home/bob/Documents/file.zip"]) + """ + elem = get_elem_by_bid(page, bid, demo_mode != "off") + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=True) + + with page.expect_file_chooser() as fc_info: + elem.click(timeout=500) + + file_chooser = fc_info.value + file_chooser.set_files(file) + + +# https://playwright.dev/python/docs/input#upload-files +def mouse_upload_file(x: float, y: float, file: str | list[str]): + """ + Click a location and wait for a "filechooser" event, then select one + or multiple input files for upload. Relative file paths are resolved + relative to the current working directory. An empty list clears the + selected files. + + Examples: + mouse_upload_file(132.1, 547, "my_receipt.pdf") + mouse_upload_file(328, 812, ["/home/bob/Documents/image.jpg", "/home/bob/Documents/file.zip"]) + """ + if demo_mode != "off": + smooth_move_visual_cursor_to(page, x, y) + highlight_by_box(page, {"x": x, "y": y, "width": 1, "height": 1}) + + with page.expect_file_chooser() as fc_info: + page.mouse.click(x, y) + + file_chooser = fc_info.value + file_chooser.set_files(file) diff --git a/core/src/browsergym/core/action/highlevel.py b/core/src/browsergym/core/action/highlevel.py index a5209acb..fc0b0826 100644 --- a/core/src/browsergym/core/action/highlevel.py +++ b/core/src/browsergym/core/action/highlevel.py @@ -20,6 +20,7 @@ focus, clear, drag_and_drop, + upload_file, scroll, mouse_move, mouse_up, @@ -27,6 +28,7 @@ mouse_click, mouse_dblclick, mouse_drag_and_drop, + mouse_upload_file, keyboard_down, keyboard_up, keyboard_press, @@ -58,6 +60,7 @@ focus, clear, drag_and_drop, + upload_file, ] COORD_ACTIONS = [ @@ -68,6 +71,7 @@ mouse_click, mouse_dblclick, mouse_drag_and_drop, + mouse_upload_file, keyboard_down, keyboard_up, keyboard_press, @@ -106,7 +110,7 @@ def __init__( ], custom_actions: Optional[list[callable]] = None, multiaction: bool = True, - demo_mode: Literal["off", "default", "only_visible_elements"] = "off", + demo_mode: Literal["off", "default", "all_blue", "only_visible_elements"] = "off", strict: bool = False, ): super().__init__(strict) @@ -209,7 +213,7 @@ def __init__( examples=examples, ) - def example_action(self, abstract: bool) -> str: + def example_action(self, abstract: bool, max_examples: int = 3) -> str: """ Returns an example action as a string. """ @@ -224,22 +228,21 @@ def example_action(self, abstract: bool) -> str: picked_examples = [] # use fill and click examples if action is present - if "fill" in self.action_set: - picked_examples.extend(self.action_set["fill"].examples) - if "click" in self.action_set: - picked_examples.extend(self.action_set["click"].examples) + for action_name in ["fill", "click", "mouse_click", "keyboard_type"]: + if action_name in self.action_set: + picked_examples.extend(self.action_set[action_name].examples) - # last resort, use all examples + # last resort, use all action examples if not picked_examples: for _, action in self.action_set.items(): - all_examples += action.examples + picked_examples += action.examples # shuffle examples rng = random.Random(1) rng.shuffle(picked_examples) if self.multiaction: - return "\n".join(picked_examples[:3]) + return "\n".join(picked_examples[:max_examples]) else: return picked_examples[0] diff --git a/core/src/browsergym/core/action/python.py b/core/src/browsergym/core/action/python.py index 6c1c9927..487c8038 100644 --- a/core/src/browsergym/core/action/python.py +++ b/core/src/browsergym/core/action/python.py @@ -24,17 +24,17 @@ def describe(self, with_long_description: bool = True, with_examples: bool = Tru ``` Here is another example: ``` -frame = page.get_by_test_id("35").frame_locator(":scope") -frame.get_by_test_id("35-776").click() +frame = page.get_by_test_id("a").frame_locator(":scope") +frame.get_by_test_id("a776").click() ``` Note that Playwright's `get_by_test_id()` method is configured to use the `bid` attribute to locate HTML elements, instead of the default `data-testid`. Also, Playwright's locators can not traverse iframes, so you have to locate parent iframes first in order to locate an element in an iframe. The `bid` attribute contains all the information -required to recursively locate an element. For example, an element with `bid="23-557-2"` can be retrieved as follows: +required to recursively locate an element. For example, an element with `bid="ac2"` can be retrieved as follows: ``` -frame = page.get_by_test_id("23").frame_locator(":scope") -frame = frame.get_by_test_id("23-557").frame_locator(":scope") -elem = frame.get_by_test_id("23-557-2") +frame = page.get_by_test_id("a").frame_locator(":scope") +frame = frame.get_by_test_id("ac").frame_locator(":scope") +elem = frame.get_by_test_id("ac2") ``` """ else: @@ -77,10 +77,10 @@ def example_action(self, abstract: bool) -> str: One single bloc of Python code. Do not include any explanation, only valid Python code.""" else: return """\ -frame = page.get_by_test_id("23").frame_locator(":scope") -frame = page.get_by_test_id("23-557").frame_locator(":scope") -frame.get_by_test_id("23-557-2").fill("Hello world!") -frame.get_by_test_id("23-557-3").click() +frame = page.get_by_test_id("b").frame_locator(":scope") +frame = page.get_by_test_id("ba").frame_locator(":scope") +frame.get_by_test_id("ba2").fill("Hello world!") +frame.get_by_test_id("ba3").click() """ def to_python_code(self, action): diff --git a/core/src/browsergym/core/action/utils.py b/core/src/browsergym/core/action/utils.py index f71a55f2..48540d6f 100644 --- a/core/src/browsergym/core/action/utils.py +++ b/core/src/browsergym/core/action/utils.py @@ -1,4 +1,5 @@ import playwright.sync_api +from typing import Literal def get_elem_by_bid( @@ -6,9 +7,9 @@ def get_elem_by_bid( ) -> playwright.sync_api.Locator: """ Parse the given bid to sequentially locate every nested frame leading to the bid, then - locate the bid element. Bids are expected to take the form "XX-...-YY-ZZ", which means - the element ZZ is located inside frame YY, which is located inside frame ..., which is - located inside frame XX, which is located inside the page's main frame. + locate the bid element. Bids are expected to take the form "abb123", which means + the element abb123 is located inside frame abb, which is located inside frame ab, which is + located inside frame a, which is located inside the page's main frame. Args: bid: the browsergym id (playwright testid) of the page element. @@ -24,33 +25,42 @@ def get_elem_by_bid( current_frame = page # dive into each nested frame, to the frame where the element is located - for i in range(bid.count("-")): - frame_bid = "-".join(bid.split("-")[: i + 1]) + i = 0 + while bid[i:] and not bid[i:].isnumeric(): + i += 1 + frame_bid = bid[:i] # bid of the next frame to select frame_elem = current_frame.get_by_test_id(frame_bid) + if not frame_elem.count(): + raise ValueError(f'could not find element with bid "{frame_bid}"') if scroll_into_view: - frame_elem.scroll_into_view_if_needed() + frame_elem.scroll_into_view_if_needed(timeout=500) current_frame = frame_elem.frame_locator(":scope") # finally, we should have selected the frame where the target element is elem = current_frame.get_by_test_id(bid) + if not elem.count(): + raise ValueError(f'Could not find element with bid "{bid}".') if scroll_into_view: - elem.scroll_into_view_if_needed() + elem.scroll_into_view_if_needed(timeout=500) return elem -def highlight_by_box(page: playwright.sync_api.Page, box: dict, is_visible: bool = True): +def highlight_by_box( + page: playwright.sync_api.Page, box: dict, color: Literal["blue", "red"] = "blue" +): """Highlights the target element based on its bounding box attributes.""" + assert color in ("blue", "red") + if box: left, top, width, height = box["x"], box["y"], box["width"], box["height"] - color = "blue" if is_visible else "red" page.evaluate( f"""\ const overlay = document.createElement('div'); document.body.appendChild(overlay); overlay.setAttribute('style', ` all: initial; - position: absolute; + position: fixed; border: 2px solid transparent; /* Start with transparent border */ borderRadius: 10px; /* Add rounded corners */ boxShadow: 0 0 0px {color}; /* Initial boxShadow with 0px spread */ @@ -111,7 +121,7 @@ def smooth_move_visual_cursor_to( `; cursor.setAttribute('style', ` all: initial; - position: absolute; + position: fixed; opacity: 0.7; /* Slightly transparent */ z-index: 2147483647; /* Maximum value */ pointer-events: none; /* Ensures the SVG doesn't interfere with page interactions */ @@ -188,35 +198,39 @@ def smooth_move_visual_cursor_to( def check_for_overlay( - page: playwright.sync_api.Page, - bid: str, - element: playwright.sync_api.ElementHandle, + page: playwright.sync_api.Page, bid: str, element: playwright.sync_api.ElementHandle, box: dict ): - """Checks in a given element is the topmost element at its center position by default. + if not element: + return False + + visibility = element.get_attribute("browsergym_visibility_ratio") + if visibility is not None: + return float(visibility) >= 0.5 + + """Checks if a given element is the topmost element at its center position by default. If check_corners is True, it checks if any of the corners is visible.""" - if element: - box = element.bounding_box() - if box: - # corners - points_to_check = [ - (box["x"], box["y"]), - (box["x"] + box["width"], box["y"]), - (box["x"], box["y"] + box["height"]), - (box["x"] + box["width"], box["y"] + box["height"]), - ] - - for x, y in points_to_check: - # Execute JavaScript to find the topmost element at the point. - top_element = page.evaluate( - f"""() => {{ - const el = document.elementFromPoint({x}, {y}); - return el ? el.outerHTML : ''; - }}""" - ) - - # Check if the topmost element is the element we're interested in. - if top_element and bid in top_element: - return True + if box: + # corners + points_to_check = [ + (box["x"], box["y"]), + (box["x"] + box["width"], box["y"]), + (box["x"], box["y"] + box["height"]), + (box["x"] + box["width"], box["y"] + box["height"]), + ] + + for x, y in points_to_check: + # Execute JavaScript to find the topmost element at the point. + top_element = page.evaluate( + f"""() => {{ + const el = document.elementFromPoint({x}, {y}); + return el ? el.outerHTML : ''; + }}""" + ) + + # Check if the topmost element is the element we're interested in. + if top_element and bid in top_element: + return True + return False @@ -224,16 +238,34 @@ def add_demo_mode_effects( page: playwright.sync_api.Page, elem: playwright.sync_api.ElementHandle, bid: str, + demo_mode: Literal["off", "default", "all_blue", "only_visible_elements"], move_cursor: bool = True, - demo_mode_type: str = "default", ): + if demo_mode == "off": + return + """Adds visual effects to the target element""" box = elem.bounding_box() + # box = extract_bounds_cdp(page, bid) if box: center_x, center_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2 - is_top_element = check_for_overlay(page, bid, elem) - - if is_top_element or demo_mode_type == "default": - if move_cursor: - smooth_move_visual_cursor_to(page, center_x, center_y) - highlight_by_box(page, box, is_visible=is_top_element) + is_top_element = check_for_overlay(page, bid, elem, box) + + if demo_mode == "only_visible_elements": + if not is_top_element: + return + else: + color = "blue" + + elif demo_mode == "default": + if is_top_element: + color = "blue" + else: + color = "red" + + elif demo_mode == "all_blue": + color = "blue" + + if move_cursor: + smooth_move_visual_cursor_to(page, center_x, center_y) + highlight_by_box(page, box, color=color) diff --git a/core/src/browsergym/core/chat.py b/core/src/browsergym/core/chat.py index 758d1489..f9221444 100644 --- a/core/src/browsergym/core/chat.py +++ b/core/src/browsergym/core/chat.py @@ -4,18 +4,20 @@ import logging import playwright.sync_api import re +import time from importlib import resources from . import _get_global_playwright, chat_files -CHATBOX_HTML_PATH = str(resources.files(chat_files).joinpath("chatbox.html")) -ASSISTANT_IMG_PATH = str(resources.files(chat_files).joinpath("assistant.png")) +CHATBOX_DIR = resources.files(chat_files) class Chat: - def __init__(self, headless: bool, chat_size=(500, 800), record_video_dir=None) -> None: + def __init__( + self, headless: bool, chat_size=(500, 800), record_video_dir=None, modern=True + ) -> None: self.messages = [] # create a new browser, browser context and page for the chat @@ -29,13 +31,17 @@ def __init__(self, headless: bool, chat_size=(500, 800), record_video_dir=None) record_video_size=dict(width=chat_size[0], height=chat_size[1]), ) self.page = self.context.new_page() + self.recording_start_time = time.time() if record_video_dir else None # setup the chat page self.page.expose_function( "send_user_message", lambda msg: self.add_message(role="user", msg=msg, from_js=True) ) - self.page.set_content(get_chatbox_html()) + if modern: + self.page.set_content(get_chatbox_modern(CHATBOX_DIR)) + else: + self.page.set_content(get_chatbox_classic(CHATBOX_DIR)) def add_message( self, role: Literal["user", "assistant", "info"], msg: str, from_js: bool = False @@ -46,8 +52,6 @@ def add_message( if role in ("user", "assistant"): self.messages.append({"role": role, "message": msg}) if not from_js: - # change new lines to html - msg = msg.replace("\n", "
") self.page.evaluate(f"addChatMessage({repr(role)}, {repr(msg)});") def wait_for_user_message(self): @@ -63,16 +67,19 @@ def close(self): self.browser.close() -def get_chatbox_html() -> str: - with open(CHATBOX_HTML_PATH, "r") as file: +def get_chatbox_modern(chatbox_dir) -> str: + with open(chatbox_dir / "chatbox_modern.html", "r") as file: chatbox_html = file.read() - with open(ASSISTANT_IMG_PATH, "rb") as f: - # image = Image.open(f) + return chatbox_html + + +def get_chatbox_classic(chatbox_dir) -> str: + with open(chatbox_dir / "chatbox.html", "r") as file: + chatbox_html = file.read() + with open(chatbox_dir / "assistant.png", "rb") as f: image_base64 = base64.b64encode(f.read()).decode("utf-8") - # hard-code the assistant image in the HTML assistant_image_url = f"data:image/png;base64,{image_base64}" chatbox_html = re.sub("", assistant_image_url, chatbox_html) - return chatbox_html diff --git a/core/src/browsergym/core/chat_files/chatbox.html b/core/src/browsergym/core/chat_files/chatbox.html index 8424ad08..2afe6e59 100644 --- a/core/src/browsergym/core/chat_files/chatbox.html +++ b/core/src/browsergym/core/chat_files/chatbox.html @@ -133,7 +133,7 @@