From 903329611be7a51e9405d3c6e0e88edbab1e41d5 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Sun, 15 Dec 2024 15:23:55 -0500 Subject: [PATCH 1/7] Update CDP Mode --- seleniumbase/core/sb_cdp.py | 35 +++++++++++++++---- .../undetected/cdp_driver/_contradict.py | 6 ++-- seleniumbase/undetected/cdp_driver/browser.py | 14 ++++---- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index 6d196672446..7cde2ee2f39 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -519,7 +519,20 @@ def __get_attribute(self, element, attribute): try: return element.get_js_attributes()[attribute] except Exception: - return None + if not attribute: + raise + try: + attribute_str = element.get_js_attributes() + locate = ' %s="' % attribute + if locate in attribute_str.outerHTML: + outer_html = attribute_str.outerHTML + attr_start = outer_html.find(locate) + len(locate) + attr_end = outer_html.find('"', attr_start) + value = outer_html[attr_start:attr_end] + return value + except Exception: + pass + return None def __get_x_scroll_offset(self): x_scroll_offset = self.loop.run_until_complete( @@ -620,11 +633,12 @@ def click_active_element(self): def click_if_visible(self, selector): if self.is_element_visible(selector): - element = self.find_element(selector) - element.scroll_into_view() - element.click() - self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + with suppress(Exception): + element = self.find_element(selector, timeout=0) + element.scroll_into_view() + element.click() + self.__slow_mode_pause_if_set() + self.loop.run_until_complete(self.page.wait()) def click_visible_elements(self, selector, limit=0): """Finds all matching page elements and clicks visible ones in order. @@ -1094,7 +1108,14 @@ def get_element_attributes(self, selector): ) def get_element_attribute(self, selector, attribute): - return self.get_element_attributes(selector)[attribute] + attributes = self.get_element_attributes(selector) + with suppress(Exception): + return attributes[attribute] + locate = ' %s="' % attribute + value = self.get_attribute(selector, attribute) + if not value and locate not in attributes: + raise KeyError(attribute) + return value def get_attribute(self, selector, attribute): return self.find_element(selector).get_attribute(attribute) diff --git a/seleniumbase/undetected/cdp_driver/_contradict.py b/seleniumbase/undetected/cdp_driver/_contradict.py index e087e4cf07e..ab6cf681c9a 100644 --- a/seleniumbase/undetected/cdp_driver/_contradict.py +++ b/seleniumbase/undetected/cdp_driver/_contradict.py @@ -31,12 +31,12 @@ class ContraDict(dict): def __init__(self, *args, **kwargs): super().__init__() - silent = kwargs.pop("silent", False) + # silent = kwargs.pop("silent", False) _ = dict(*args, **kwargs) super().__setattr__("__dict__", self) for k, v in _.items(): - _check_key(k, self, False, silent) + _check_key(k, self, False, True) super().__setitem__(k, _wrap(self.__class__, v)) def __setitem__(self, key, value): @@ -90,7 +90,7 @@ def _wrap(cls, v): def _check_key( - key: str, mapping: _Mapping, boolean: bool = False, silent=False + key: str, mapping: _Mapping, boolean: bool = False, silent=True ): """Checks `key` and warns if needed. :param key: diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index cadb8fa73bf..83ed2c133a9 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -10,6 +10,7 @@ import pickle import re import shutil +import time import urllib.parse import urllib.request import warnings @@ -30,8 +31,6 @@ def get_registered_instances(): def deconstruct_browser(): - import time - for _ in __registered__instances__: if not _.stopped: _.stop() @@ -117,8 +116,13 @@ async def create( port=port, **kwargs, ) - instance = cls(config) - await instance.start() + try: + instance = cls(config) + await instance.start() + except Exception: + time.sleep(0.15) + instance = cls(config) + await instance.start() return instance def __init__(self, config: Config, **kwargs): @@ -379,8 +383,6 @@ async def start(self=None) -> Browser: -------------------------------- Failed to connect to the browser -------------------------------- - Possibly because you are running as "root". - If so, you may need to use no_sandbox=True. """ ) ) From 57e7b2bea83e854891fe904986b896af02eabe0b Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Sun, 15 Dec 2024 15:25:02 -0500 Subject: [PATCH 2/7] Update Firefox options and extensions --- seleniumbase/core/browser_launcher.py | 2 -- seleniumbase/extensions/ReadMe.md | 3 ++- seleniumbase/extensions/firefox_addon.xpi | Bin 0 -> 8038 bytes 3 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 seleniumbase/extensions/firefox_addon.xpi diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index c2d9ae5b140..eb50a898daf 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -2407,8 +2407,6 @@ def _set_firefox_options( options.set_preference("dom.webnotifications.enabled", False) options.set_preference("dom.disable_beforeunload", True) options.set_preference("browser.contentblocking.database.enabled", True) - options.set_preference("extensions.allowPrivateBrowsingByDefault", True) - options.set_preference("extensions.PrivateBrowsing.notification", False) options.set_preference("extensions.systemAddon.update.enabled", False) options.set_preference("extensions.update.autoUpdateDefault", False) options.set_preference("extensions.update.enabled", False) diff --git a/seleniumbase/extensions/ReadMe.md b/seleniumbase/extensions/ReadMe.md index 3b528c91dca..3b93bc7cfe9 100644 --- a/seleniumbase/extensions/ReadMe.md +++ b/seleniumbase/extensions/ReadMe.md @@ -9,4 +9,5 @@ * ad_block.zip => This extension blocks certain types of iframe ads from loading. * disable_csp.zip => This extension disables a website's Content-Security-Policy. * recorder.zip => Save browser actions to sessionStorage with good CSS selectors. -* sbase_ext.zip => An extension that does nothing, but helps people learn things. +* sbase_ext.zip => A Chromium extension that does nothing. (Used for testing purposes) +* firefox_addon.xpi => A Firefox add-on that does nothing. (Used for testing purposes) diff --git a/seleniumbase/extensions/firefox_addon.xpi b/seleniumbase/extensions/firefox_addon.xpi new file mode 100644 index 0000000000000000000000000000000000000000..dca8e2e12312f55a570208802b504febadd24052 GIT binary patch literal 8038 zcmaKxWmFyAvaWGmxFxs*2<{Nv-F@K@+zA8-?(QzZLU4C?3y|RM?y`V|>mlRZPw&~g z=IAlHXZNh0RnZpDkhAjd=6PGH6jI?XNKzf%d`>7?auW3$77+ZD=SFkZS+-GCT_PL>=e}IH&s*bX z$#dZI{fFYAo^^?*i=|c$Ux)NY*W^b~$L3n@eg`xJ#LJ74EIfkt{PB7E>vs2F52gPY z5X$RaTO&J5GgGJEA=%pnD)TGW38A0f;Uv&7F`cC!71^*?H&ihPcd-)o61AF`!~6<* zu>;1k;VyQaf;q-aDls5jf_Boyz(m% z3=jeu$XT*H^qy`_bXbvK)%dyw7a=syJWz`Q{P^Zba^%0AiR|=N1W(+tRRu|2tF_X2 ztu7y3>Nq`SFPEo&(ipPpxW@do&H)9{6l_1QBQMHD?i*#rq?JxQDH?hQXTJ~E;Z;5V z9J*fRF*(eL^1{6+2XyB`y3=)M)vX;0Qd`JM&ZsBK8q8Sn))Oi1G$1FAzFA^nsp$1O zCLGCz{3CCdTiq^CuUR>Ko%zps!+m{1UR+I>L0UnA$=Ke>l<|)&CXS9M1PEaVZ#?P3 zdQ4qdTS^0!$jBQ2Nruxa5YWFMpdG8QD_3Jn2sG^!MmL}0_oQIx-ba?2^FSO0QWjgo zev7S}j!#`@CzvCQ{W(nOm|jCL53Dzot_BtToUr4vxYL{)h*yyZXchWwntm919}kcdBl5GYm{-0FDwmjJS!Cgz!%_At4E2Ym5je5u6Mq3^7=o_wj2; z^UsSN6*p@-_U#XIji7}!J@f3=YXRuXH?+ea@W^s<0eFd~*$gdYI@-FZnkPsiVV z`5YG{;)3=g_+W;lW&IHR>4Cw4ycDs()LIH*=pt!f0IuvtaCJ0szU9fr#bYxces=#3 zdILmT9&JSblRiiOyyXJ8Nx%<)DEu>S=ZDd!$2!8aHd5RGX&|(QEMr%-3?E}kjQRN4 zvrp_fJ11V5YWF_0KO`w8pnG$x!heDqLTDVm&z{LQ=S*;s0(GDMs)@(2xF-kW(n3H$ zSHYLq;{eNDH;crG0SNxFpDZjx=haLGDHl8h!iGyh39NGV$zpoP0E^edtRjSv;D=}w zJ3WFK)7G?oBtc1jdy|R9kZ_v43iFm@vnNEmH+Wytazr6#evAKLEAcZ0b!0-B z3K$ogbyCatGkGgVm9MaSgN_Wi^T$NQ)vh!1@6gyG=tX;&f*&DiW zHgFlXei7-`Yd8$ej($d`ww?Bt4(#CF-^;)F;xl4(mPxE<@THvn1Omfsjn|)U8q@>(bD{Jp|Go$kT3rcg5ERV zP2Tlff76~wAsWES0XasH$@l{RZolxdU@^uO&m~L&1o9A#wEoC_Q=<1_o;LOcS zYF!6#EJ`9uSR(qw)y%PX6n(-#vQo<9)_UFkCD*xZ1=$pa;A^% z@XKYAZSXbU7Ft*Fv#dBmX0J7)zO>`I6#5+St!X7Wx2<$sl)ouA89&+Mi0JTIf9t~h zZ1Q865evj}ArC=6kK5W z8Yz0lb7{OiL{!iT3*~!AaYP}b9hzk!)a-#xi{`4-ZWutUL3{+eo=vzfERrme>9YZj zA3!RCcKfVbj)yI9Y#XMkDFeb20{lLq7s+HRjf}nFMVU$<)Z*f#X`U6wFs>+yJi^ZS zO`sSP@j5zEhYpevu?5ZCwr4*Vg;|MLi~g69-5>33z@u@JQxrKAf*pz)sdk!;UBtGP zyLY`7nWj25Q({jKQF_c-Ik?ni7coBA-!x zdqiQ#4LIFEg$_7@ZQHFFFlmbTQ~>~af*AF~eY|I=j$b8lrWp13pb2Jx@R|ha zJ*%aO&;WqUUB@R=kRn&XN&mr`D^do(YSg3CGbb#tW0X(djUEj~4+ z>dWu&MxlJ!3Ovm3UU~p)aqBibju9w4XfDY+*E5YR!ujJfTQ@VgDosAxswbnh?&Qq~ z`Rf+L0YZ2&nYG+tFGpwjj#VqsD5KxS7b(CcJeyC_+KBGCZ*`Bs&sMaD1r>3s4bQF9W;H>FR z=fPynPLM);s{*!~Cvem?KW&KcE*kwQuIVyh<kl1gx_RLt??!o1La5s$bJ#Ok<9wnBN# zn_g8@GZv(b%wR7IZTpNn3sNbJePY|NN^<>3U93@*XB`ckyFqJhx0pbCZ{Q>m#M-vS zo%=+~PZJgtVoO)RKp!<39!LK$mMs>Ru_f*=%geHGs`lAn=KmG_e^s0^60ROFL z#{PRB_(!|UXls@@Dj$=}O!(uRx3cF!Ojt-53r$>@I59-sjL1G8*P*|RUSbJy4r}B;_(43D1U4q3*Qca3ZWtl<=?@?y)f_^t3 z6M5j~yF$U_6Nxt$Sh%2uZd02ie|Gp~%Fo~vg9tCj_qRZ)wE#(-!d;$$;V=f%^+T3O zoVOZ{xT6B1^EAH(#g(-%x(Sr&z18-7wlQ!r+TYb4{HR-aYjD7DhlC=MAWL0$Y|_f; z9={P)x(cO{WqH|rLFZN`{*0Sq53iSTsFxtFznTa$~ae zGz>B`D6$N+qrb*#RhXyQ){(9q*r%O`}Ru9+siKl4q-ewPa!C=$$q}$4$gM#UMUDH34p)9ucn}S{$ln>>Ot%9d20R z8Xl%6>|>>VJkCtn)VBHA5Bn<3|2GInqrI-c*Dc1r9{*Xc{qxiNe?d4p8BM2{tEn!u z4oK%?VdT-0QmOTKpx}-%nJ#J zlSleuwcGjLgeV)^E638^~922k>G|#+woJCIchmx|}`2cRB^hTg&4{0_f zVs7*d5+McJb%lu)A#~+n`rd@V#`kWI!OG>~8TOPr8Y&@R4^!Uvk#9cXB9j{eC#67aFB(mIRY| z^d(~(t(y{tgR?IjVgsd{rq5m)`P-y)hgw>1MY9X*}3@PY`0t24{>Qq3!R#Fm zfw>ymTUqGJ*`JnPjP)`AIV!>Zg|^Qv_LD9b}2J z5Wyc+Z~(}w0(0Cpfaw(Bc2x5>qOS()$fm|hsP0xCR&Nsj zAmgb4b6ec80w*qdLPJ`2!Q{a}gNxO6a$ZTldfly+20&0b1FuAwvUb!L)w!RpSBEHT z%M@xpYi&6Acyc`+_oz7zu2)Z{l0G0%NN~3$wLLPq%amAVL`8^RqYK?+bCate6cNwo z31)_BhQ^gZug9pwNCwVtw)?k4M_-}E$b%*Mv%`&8^JumEl^%QiJe0#CNA)%P_sksD ztC#{=cha~iaNZg2eusk_O_J8~Oq^#Z*vG8iwvdYbDnMfMY;!2wl>$V>n8PBYEc!%; zlWm#~JE@wvFn#TbIApKp$a!zOmkK?0zA5Z9WQ1#fMt8~x4I+EwpdtLZIX6i2Rygo$ z-(i@n%*Slr5ys(%HNP4AF$d$IL!Kjgcwrz zCH@n38+`XqWKh@?l)e@57c}}Xp{|4>;zeKvUXU2X7bcYTeDU!CDEjfo5Q(RlHF%pX zx&~WRFN_Cz2x|k~N1>r=r%$V8eV^w-Red<5Ra{Hd6kCJ@6!`}$7XuU!BWa86`BnxG z-LeIy-Tl58;I9a9SLEa-v`-O`h}>a6mpLS!~?YuCy|8i&~r`)W~)~8xo*7zGEAk zo>V*#R2tsflhnjndLtKifUeAWb9eX+h^Ai{J7CY^T1dc}<<69&i5gVQ_O_5vKeGEG zj%SESWPu+`s&XSUL)8UC%WUrxMIyj@W<5Ex>}>H!WhLP%tW#`0ng_pD!`&5C4eVec zIDGNzV+VK8o7HrTnhVD_nMi_$G7q_99R%>A?q4jDE3InKb|a~pvn@3zLo~LZJ~nzk zSc(YWf)|`@-JR9$!_%hb&K3EFPSu1nz5MtwCV<`n(hT`W);G~M%832+2LQbL19qY{ zg*vB+%IvmYpjDC&K@W1PGUKYwM_%l5#-<1f-VM%S<7HG{c4P3cm63AE@<^fnSA8%; zEW`4Bj-4IXf{{qJo#oes?oI&>!%ODI3UudP)@1eP(bh$BfhD&ib_ojo?Ee)CSoAwPVASM#9OT!n5nBkpbUn8n}*bYUav}D ze-OtOEjEVivgK}DOq-k^x-!e4kokueZMNas}b2AYaFoF?4{G4|P zeF|@X4%cRr^5)U1#${LhR^SM@vB6<7(t9)+tgR3+SLw1$+c_3os53Ywf|GmFY<0R| zPKip7*G$W-!0(7k7&R*|5S!dc^(AbFcgGLe0<~Fqpp#=WwuQm7hyJ1$T$iQzsTi8% zaXx=z%mNLhGt8yC<381OcZ&s9Pg7(*>2cRY!B+=Q?ukJ8$YlE(aBNAT%IyMzl?e++4IoWco^YQOtI7DU)gZg96d&!$9*XGjxrdpQTC1NvcKhV`WG7?y=l) ztgBG$KPf2C^+Ua7xY8TdDJVDIZJKpEX@o_%AL_?p$XZeUP z-1SYVY)B(0^J0bonSLiColToWnt$Ll?F$R1(nSOEFVk1NM z6?G(uVhibGAP`=6s0(dF%Oq}s54Z!Bhma*OZnb%jGSI#r){qO1MV7eaO>-a~P7~#P;W-_xr&8Zgl?klZ$!$^kFHs_l4T?M`9MKwWmo|-J)E-7z*g#SX3wN zH^eu;tF!I#T{Je;-CGtXy$b%;rmUIk-gqX=}E`C@|$ev={wzTQmWm4V4P8>5#SXmKH# z4J1@U5S>cre4fd)tl-YUX4{iB#$!R+qRF8M0z+oy1(fYnao%iU(5u&#Qcbv2#cC*i zm_$?UNL-HKiM*F<8@-02fv4W8l1rsB>~T;+fMG6q=D)pLqQBcaOWQ)~^FH=BdJ%~5 zKM|xwpg$}mixz@$6fAKlL(pyV7+RW1d}?dYpd+dvz39hLN1Qc)@{-|EVax{T@a)8|YtmS$^iM;63A@@QE7Vs1{R#gLp- z9ub_=Rmn$AJa9jnHHxaVgtD@&26HXPNUQ4ULkG9&h2&Q=z=F}Lt7l)m|1Z_C z2EoYZSeHa#&ZSPY^}^LY1yo$VE0VSS({5+o>Df08DsfE^afi$M_T%daO-mU%*UTb8879rqbi1M5H ztX!nA0}jsiWBPQQgp5SYLcPowqBtt54EF>Ud{G!hhmfiKtGS0JC;`+rw;E60+jsSa z{TvjJ6*c+`j4L-4781I5VrvonEypt?rZOLhjKUlCm9wncj2s?0oUN^mcUKERH(3{v z6%oX=A?g^9lr4`_Fr_yJsnsJ_k$H?H1?lXKgyWN4lKCxE^h@+ObYo3gW_E$acmP!1-fMAW1<5?kj^oVO)n# z6=KFKqr`k=NJ_GhP}mTEaoN8E&fi@2pY|uE{r9rJ^S$5n^DmotJ^k~y#BUb*@1=jo zUB3a~UxxT9q5p3j_-`M7|5frEjQwRCujuxlK3*}|znA}Av;P*ge;NO4Fn=!pGYng^ ze|z}5VE(P`|1vP@-#z@P^#ASP?=|`77Mjrh4+pO*TS*oc?)PipUq{PpZIg+9U;Pgk CGK-S{ literal 0 HcmV?d00001 From 0d6c255dfdc3bf3bd95e7f6ea70dda89889b3e37 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Sun, 15 Dec 2024 15:26:09 -0500 Subject: [PATCH 3/7] Add method to get element at coordinates --- help_docs/method_summary.md | 2 ++ seleniumbase/fixtures/base_case.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md index bc3a47e9931..6a27bde9f43 100644 --- a/help_docs/method_summary.md +++ b/help_docs/method_summary.md @@ -227,6 +227,8 @@ self.execute_async_script(script, timeout=None) self.safe_execute_script(script, *args, **kwargs) +self.get_element_at_x_y(x, y) + self.get_gui_element_rect(selector, by="css selector") self.get_gui_element_center(selector, by="css selector") diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index bbb610ce212..d93fa380625 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -3416,6 +3416,14 @@ def safe_execute_script(self, script, *args, **kwargs): self.activate_jquery() return self.driver.execute_script(script, *args, **kwargs) + def get_element_at_x_y(self, x, y): + """Return element at current window's x,y coordinates.""" + self.__check_scope() + self._check_browser() + return self.execute_script( + "return document.elementFromPoint(%s, %s);" % (x, y) + ) + def get_gui_element_rect(self, selector, by="css selector"): """Very similar to element.rect, but the x, y coordinates are relative to the entire screen, rather than the browser window. From 6cb884422fc44b529687b8f75852fa30dfe32977 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Sun, 15 Dec 2024 15:27:40 -0500 Subject: [PATCH 4/7] Update examples --- examples/ReadMe.md | 2 +- examples/cdp_mode/ReadMe.md | 53 ++++++++++-- examples/cdp_mode/raw_planetmc.py | 2 +- examples/cdp_mode/raw_tiktok.py | 9 +- examples/presenter/uc_presentation_3.py | 2 +- examples/raw_google.py | 8 ++ examples/test_download_files.py | 2 +- examples/test_get_swag.py | 20 +++++ examples/translations/italian_test_1.py | 8 +- help_docs/syntax_formats.md | 108 ++++++++++++++++++++++-- 10 files changed, 187 insertions(+), 27 deletions(-) create mode 100644 examples/raw_google.py create mode 100644 examples/test_get_swag.py diff --git a/examples/ReadMe.md b/examples/ReadMe.md index 0f3102fc0c2..a8381af1805 100644 --- a/examples/ReadMe.md +++ b/examples/ReadMe.md @@ -6,7 +6,7 @@ * SeleniumBase tests are run with pytest. * Chrome is the default browser if not specified. -* Tests are structured using [23 unique syntax formats](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/syntax_formats.md). +* Tests are structured using [25 unique syntax formats](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/syntax_formats.md). * Logs from test failures are saved to ``./latest_logs/``. * Tests can be run with [multiple command-line options](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/customizing_test_runs.md). * Examples can be found in: **[SeleniumBase/examples/](https://github.com/seleniumbase/SeleniumBase/tree/master/examples)**. diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 5815ba409b5..ddef88e9100 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -2,7 +2,7 @@ ## [](https://github.com/seleniumbase/SeleniumBase/) CDP Mode 🐙 -🐙 SeleniumBase CDP Mode (Chrome Devtools Protocol Mode) is a special mode inside of SeleniumBase UC Mode that lets bots appear human while controlling the browser with the CDP-Driver. Although regular UC Mode can't perform WebDriver actions while the driver is disconnected from the browser, the CDP-Driver can still perform actions while maintaining its cover. +🐙 SeleniumBase CDP Mode (Chrome Devtools Protocol Mode) is a special mode inside of SeleniumBase UC Mode that lets bots appear human while controlling the browser with the CDP-Driver. Although regular UC Mode can't perform WebDriver actions while the driver is disconnected from the browser, the CDP-Driver can. -------- @@ -13,7 +13,7 @@ 👤 UC Mode avoids bot-detection by first disconnecting WebDriver from the browser at strategic times, calling special PyAutoGUI methods to bypass CAPTCHAs (as needed), and finally reconnecting the driver afterwards so that WebDriver actions can be performed again. Although this approach works for bypassing simple CAPTCHAs, more flexibility is needed for bypassing bot-detection on websites with advanced protection. (That's where CDP Mode comes in.) -🐙 CDP Mode is based on python-cdp, trio-cdp, and nodriver. trio-cdp is an early implementation of python-cdp, and nodriver is a modern implementation of python-cdp. (Refactored Python-CDP code is imported from MyCDP.) +🐙 CDP Mode is based on python-cdp, trio-cdp, and nodriver. trio-cdp is an early implementation of python-cdp, and nodriver is a modern implementation of python-cdp. (Refactored Python-CDP code is imported from MyCDP.) 🐙 CDP Mode includes multiple updates to the above, such as: @@ -35,17 +35,52 @@ That disconnects WebDriver from Chrome (which prevents detection), and gives you access to `sb.cdp` methods (which don't trigger anti-bot checks). +Simple example: ([SeleniumBase/examples/cdp_mode/raw_planetmc.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_planetmc.py)) + +```python +from seleniumbase import SB + +with SB(uc=True, test=True) as sb: + url = "www.planetminecraft.com/account/sign_in/" + sb.activate_cdp_mode(url) + sb.sleep(2) + sb.cdp.gui_click_element("#turnstile-widget div") + sb.sleep(2) +``` + + + +(If the CAPTCHA wasn't initially bypassed, then the click gets the job done.) + +Note that `PyAutoGUI` is an optional dependency. If calling a method that uses it when not already installed, then `SeleniumBase` will install it at run-time, which pauses the script briefly. + +For standard Cloudflare pages, use `sb.uc_gui_click_captcha()` if Turnstiles aren't initially bypassed. Example: ([SeleniumBase/examples/cdp_mode/raw_gitlab.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_gitlab.py)) + +```python +from seleniumbase import SB + +with SB(uc=True, test=True, locale_code="en") as sb: + url = "https://gitlab.com/users/sign_in" + sb.activate_cdp_mode(url) + sb.uc_gui_click_captcha() + sb.sleep(2) +``` + + + +-------- + ### 🐙 Here are a few common `sb.cdp` methods: -* `sb.cdp.click(selector)` +* `sb.cdp.click(selector)` (Uses the CDP API to click) * `sb.cdp.click_if_visible(selector)` -* `sb.cdp.gui_click_element(selector)` +* `sb.cdp.gui_click_element(selector)` (Uses `PyAutoGUI`) * `sb.cdp.type(selector, text)` -* `sb.cdp.press_keys(selector, text)` +* `sb.cdp.press_keys(selector, text)` (Human-speed `type`) * `sb.cdp.select_all(selector)` * `sb.cdp.get_text(selector)` -When `type()` is too fast, use the slower `press_keys()` to avoid detection. You can also use `sb.sleep(seconds)` to slow things down. Methods that start with `sb.cdp.gui` use `PyAutoGUI` for interaction. +Methods that start with `sb.cdp.gui` use `PyAutoGUI` for interaction. To use WebDriver methods again, call: @@ -57,15 +92,17 @@ To disconnect again, call: * **`sb.disconnect()`** -While disconnected, if you accidentally call a WebDriver method, then SeleniumBase will attempt to use the CDP Mode version of that method (if available). For example, if you accidentally call `sb.click(selector)` instead of `sb.cdp.click(selector)`, then your WebDriver call will automatically be redirected to the CDP Mode version. Not all WebDriver methods have a matching CDP Mode method. In that scenario, calling a WebDriver method while disconnected could raise an error, or make WebDriver automatically reconnect first. +While disconnected, if you accidentally call a WebDriver method, then SeleniumBase will attempt to use the CDP Mode version of that method (if available). For example, if you accidentally call `sb.click(selector)` instead of `sb.cdp.click(selector)`, then your WebDriver call will automatically be redirected to the CDP Mode version. Not all WebDriver methods have a matching CDP Mode method. In that scenario, calling a WebDriver method while disconnected could raise an error, or make WebDriver automatically reconnect first. To find out if WebDriver is connected or disconnected, call: * **`sb.is_connected()`** +Note: When CDP Mode is initialized from UC Mode, the WebDriver is disconnected from the browser. (The stealthy CDP-Driver takes over.) + -------- -### 🐙 CDP Mode Examples ([SeleniumBase/examples/cdp_mode](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode)) +### 🐙 CDP Mode examples ([SeleniumBase/examples/cdp_mode](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode))

diff --git a/examples/cdp_mode/raw_planetmc.py b/examples/cdp_mode/raw_planetmc.py index 79647f779e2..d80ef551178 100644 --- a/examples/cdp_mode/raw_planetmc.py +++ b/examples/cdp_mode/raw_planetmc.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True) as sb: url = "www.planetminecraft.com/account/sign_in/" sb.activate_cdp_mode(url) sb.sleep(2) diff --git a/examples/cdp_mode/raw_tiktok.py b/examples/cdp_mode/raw_tiktok.py index b685f9171a3..391a0e2a43f 100644 --- a/examples/cdp_mode/raw_tiktok.py +++ b/examples/cdp_mode/raw_tiktok.py @@ -1,15 +1,12 @@ from seleniumbase import SB -with SB( - uc=True, test=True, locale_code="en", incognito=True, ad_block=True -) as sb: +with SB(uc=True, test=True, incognito=True, ad_block=True) as sb: url = "https://www.tiktok.com/@startrek?lang=en" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.click('button:contains("Refresh")') + sb.cdp.click_if_visible('button:contains("Refresh")') sb.sleep(1.5) print(sb.cdp.get_text('h2[data-e2e="user-bio"]')) - for i in range(55): + for i in range(50): sb.cdp.scroll_down(12) - sb.sleep(0.05) sb.sleep(1) diff --git a/examples/presenter/uc_presentation_3.py b/examples/presenter/uc_presentation_3.py index ac9f6b3e610..3f6d6ca2536 100644 --- a/examples/presenter/uc_presentation_3.py +++ b/examples/presenter/uc_presentation_3.py @@ -456,7 +456,7 @@ def test_presentation_3(self): "

" "

" "There are different ways of stucturing SeleniumBase scripts." - ' (Internally called: "The 23 Syntax Formats")' + ' (Internally called: "The 25 Syntax Formats")' "

" 'Most examples use Syntax Format 1: "BaseCase direct class' ' inheritance", which uses the "pytest" test runner.' diff --git a/examples/raw_google.py b/examples/raw_google.py new file mode 100644 index 00000000000..c56fff6d5ee --- /dev/null +++ b/examples/raw_google.py @@ -0,0 +1,8 @@ +from seleniumbase import SB + +with SB(test=True, ad_block=True, locale_code="en") as sb: + sb.open("https://google.com/ncr") + sb.type('[title="Search"]', "SeleniumBase GitHub page\n") + sb.click('[href*="github.com/seleniumbase/SeleniumBase"]') + sb.save_screenshot_to_logs() # (See ./latest_logs folder) + print(sb.get_page_title()) diff --git a/examples/test_download_files.py b/examples/test_download_files.py index f10f1e47947..867d9e79480 100644 --- a/examples/test_download_files.py +++ b/examples/test_download_files.py @@ -31,7 +31,7 @@ def test_download_files_from_pypi(self): self.assert_text("Download files", "a#files-tab") pkg_header = self.get_text("h1.package-header__name").strip() pkg_name = pkg_header.replace(" ", "-") - whl_file = pkg_name + "-py2.py3-none-any.whl" + whl_file = pkg_name + "-py3-none-any.whl" tar_gz_file = pkg_name + ".tar.gz" # Click the links to download the files into: "./downloaded_files/" diff --git a/examples/test_get_swag.py b/examples/test_get_swag.py new file mode 100644 index 00000000000..6d0d71314f3 --- /dev/null +++ b/examples/test_get_swag.py @@ -0,0 +1,20 @@ +from seleniumbase import BaseCase +BaseCase.main(__name__, __file__) + + +class MyTestClass(BaseCase): + def test_swag_labs(self): + self.open("https://www.saucedemo.com") + self.type("#user-name", "standard_user") + self.type("#password", "secret_sauce\n") + self.assert_element("div.inventory_list") + self.click('button[name*="backpack"]') + self.click("#shopping_cart_container a") + self.assert_text("Backpack", "div.cart_item") + self.click("button#checkout") + self.type("input#first-name", "SeleniumBase") + self.type("input#last-name", "Automation") + self.type("input#postal-code", "77123") + self.click("input#continue") + self.click("button#finish") + self.assert_text("Thank you for your order!") diff --git a/examples/translations/italian_test_1.py b/examples/translations/italian_test_1.py index f27dd5100df..d21bc2c0eaf 100644 --- a/examples/translations/italian_test_1.py +++ b/examples/translations/italian_test_1.py @@ -8,12 +8,12 @@ def test_esempio_1(self): self.apri("https://it.wikipedia.org/wiki/") self.verificare_testo("Wikipedia") self.verificare_elemento('a[title="Lingua italiana"]') - self.digitare("#searchInput", "Pizza") - self.fare_clic("#searchButton") + self.digitare('input[name="search"]', "Pizza") + self.fare_clic("#searchform button") self.verificare_testo("Pizza", "#firstHeading") self.verificare_elemento('figure img[src*="pizza"]') - self.digitare("#searchInput", "Colosseo") - self.fare_clic("#searchButton") + self.digitare('input[name="search"]', "Colosseo") + self.fare_clic("#searchform button") self.verificare_testo("Colosseo", "#firstHeading") self.verificare_elemento('figure img[src*="Colosseo"]') self.indietro() diff --git a/help_docs/syntax_formats.md b/help_docs/syntax_formats.md index eff59a301a0..18d9202f864 100644 --- a/help_docs/syntax_formats.md +++ b/help_docs/syntax_formats.md @@ -2,7 +2,7 @@ -

The 23 Syntax Formats / Design Patterns

+

The 25 Syntax Formats / Design Patterns

🔠 SeleniumBase supports multiple ways of structuring tests:

@@ -32,6 +32,8 @@
  • 21. SeleniumBase SB (Python context manager)
  • 22. The driver manager (via context manager)
  • 23. The driver manager (via direct import)
    • +
  • 24. CDP driver (async/await API. No Selenium)
    • +
  • 25. CDP driver (SB-CDP sync API. No Selenium)
    • @@ -550,12 +552,12 @@ class MiaClasseDiTest(CasoDiProva): self.apri("https://it.wikipedia.org/wiki/") self.verificare_testo("Wikipedia") self.verificare_elemento('a[title="Lingua italiana"]') - self.digitare("#searchInput", "Pizza") - self.fare_clic("#searchButton") + self.digitare('input[name="search"]', "Pizza") + self.fare_clic("#searchform button") self.verificare_testo("Pizza", "#firstHeading") self.verificare_elemento('figure img[src*="pizza"]') - self.digitare("#searchInput", "Colosseo") - self.fare_clic("#searchButton") + self.digitare('input[name="search"]', "Colosseo") + self.fare_clic("#searchform button") self.verificare_testo("Colosseo", "#firstHeading") self.verificare_elemento('figure img[src*="Colosseo"]') self.indietro() @@ -876,6 +878,21 @@ with SB(test=True, rtf=True, demo=True) as sb: (See examples/raw_test_scripts.py for the test.) +Here's another example, which uses [CDP Mode](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/ReadMe.md) from the SeleniumBase SB format: + +```python +from seleniumbase import SB + +with SB(uc=True, test=True) as sb: + url = "www.planetminecraft.com/account/sign_in/" + sb.activate_cdp_mode(url) + sb.sleep(2) + sb.cdp.gui_click_element("#turnstile-widget div") + sb.sleep(2) +``` + +(See examples/cdp_mode/raw_planetmc.py for the test.) +

    22. The driver manager (via context manager)

    @@ -994,6 +1011,87 @@ The ``Driver()`` manager format can be used as a drop-in replacement for virtual When using the ``Driver()`` format, you may need to activate a Virtual Display on your own if you want to run headed tests in a headless Linux environment. (See https://github.com/mdmintz/sbVirtualDisplay for details.) One such example of this is using an authenticated proxy, which is configured via a Chrome extension that is generated at runtime. (Note that regular headless mode in Chrome doesn't support extensions.) + +

    24. CDP driver (async/await API. No Selenium)

    + +This format provides a pure CDP way of using SeleniumBase (without Selenium or a test runner). The async/await API is used. Here's an example: + +```python +import asyncio +import time +from seleniumbase.undetected import cdp_driver + + +async def main(): + driver = await cdp_driver.cdp_util.start_async() + page = await driver.get("about:blank") + await page.set_locale("en") + await page.get("https://www.priceline.com/") + time.sleep(3) + print(await page.evaluate("document.title")) + element = await page.select('[data-testid*="endLocation"]') + await element.click_async() + time.sleep(1) + await element.send_keys_async("Boston") + time.sleep(2) + +if __name__ == "__main__": + loop = asyncio.new_event_loop() + loop.run_until_complete(main()) +``` + +(See examples/cdp_mode/raw_async.py for the test.) + + +

    25. CDP driver (SB-CDP sync API. No Selenium)

    + +This format provides a pure CDP way of using SeleniumBase (without Selenium or a test runner). The expanded SB-CDP sync API is used. Here's an example: + +```python +import asyncio +from seleniumbase.core import sb_cdp +from seleniumbase.undetected import cdp_driver + + +def main(): + url0 = "about:blank" # Set Locale code from here first + url1 = "https://www.priceline.com/" # (The "real" URL) + loop = asyncio.new_event_loop() + driver = cdp_driver.cdp_util.start_sync() + page = loop.run_until_complete(driver.get(url0)) + sb = sb_cdp.CDPMethods(loop, page, driver) + sb.set_locale("en") # This test expects English locale + sb.open(url1) + sb.sleep(2.5) + sb.internalize_links() # Don't open links in a new tab + sb.click("#link_header_nav_experiences") + sb.sleep(3.5) + sb.remove_elements("msm-cookie-banner") + sb.sleep(1.5) + location = "Amsterdam" + where_to = 'div[data-automation*="experiences"] input' + button = 'button[data-automation*="experiences-search"]' + sb.gui_click_element(where_to) + sb.press_keys(where_to, location) + sb.sleep(1) + sb.gui_click_element(button) + sb.sleep(3) + print(sb.get_title()) + print("************") + for i in range(8): + sb.scroll_down(50) + sb.sleep(0.2) + cards = sb.select_all('h2[data-automation*="product-list-card"]') + for card in cards: + print("* %s" % card.text) + + +if __name__ == "__main__": + main() +``` + +(See examples/cdp_mode/raw_cdp.py for the test.) + --------

    From fec129a4b50ad216daed06ef5e449648c82fbce7 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Sun, 15 Dec 2024 15:27:57 -0500 Subject: [PATCH 5/7] Update the ReadMe --- README.md | 123 +++++++++++++++++++++++++----------------------------- 1 file changed, 57 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 8ae5587e7ba..8d62d968084 100755 --- a/README.md +++ b/README.md @@ -60,21 +60,60 @@ 📚 Learn from [**over 200 examples** in the **SeleniumBase/examples/** folder](https://github.com/seleniumbase/SeleniumBase/tree/master/examples). -👤 Note that SeleniumBase UC Mode (Stealth Mode) has its own ReadMe. +🐙 Note that UC Mode and CDP Mode (Stealth Mode) have separate docs. -🐙 Also note that Seleniumbase CDP Mode has its own separate ReadMe. +ℹī¸ Most scripts run with raw python, although some scripts use Syntax Formats that expect pytest (a Python unit-testing framework included with SeleniumBase that can discover, collect, and run tests automatically). -ℹī¸ Scripts can be called via python, although some Syntax Formats expect pytest (a Python unit-testing framework included with SeleniumBase that can discover, collect, and run tests automatically). +-------- -

    📗 Here's my_first_test.py, which tests login, shopping, and checkout:

    +

    📗 Here's raw_google.py, which performs a Google search:

    -```bash -pytest my_first_test.py +```python +from seleniumbase import SB + +with SB(test=True, ad_block=True, locale_code="en") as sb: + sb.open("https://google.com/ncr") + sb.type('[title="Search"]', "SeleniumBase GitHub page\n") + sb.click('[href*="github.com/seleniumbase/SeleniumBase"]') + sb.save_screenshot_to_logs() # (See ./latest_logs folder) + print(sb.get_page_title()) ``` -SeleniumBase Test +> `python raw_google.py` -> ``pytest`` uses ``--chrome`` by default unless set differently. +SeleniumBase Test + +-------- + +

    📗 Here's test_get_swag.py, which tests a fake shopping site:

    + +```python +from seleniumbase import BaseCase +BaseCase.main(__name__, __file__) # Call pytest + +class MyTestClass(BaseCase): + def test_swag_labs(self): + self.open("https://www.saucedemo.com") + self.type("#user-name", "standard_user") + self.type("#password", "secret_sauce\n") + self.assert_element("div.inventory_list") + self.click('button[name*="backpack"]') + self.click("#shopping_cart_container a") + self.assert_text("Backpack", "div.cart_item") + self.click("button#checkout") + self.type("input#first-name", "SeleniumBase") + self.type("input#last-name", "Automation") + self.type("input#postal-code", "77123") + self.click("input#continue") + self.click("button#finish") + self.assert_text("Thank you for your order!") +``` + +> `pytest test_get_swag.py` + +SeleniumBase Test + +> (The default browser is ``--chrome`` if not set.) -------- @@ -168,7 +207,7 @@ With raw Selenium, that requires more code:

    📚 Learn about different ways of writing tests:

    -

    📘📝 Here's test_simple_login.py, which uses BaseCase class inheritance, and runs with pytest or pynose. (Use self.driver to access Selenium's raw driver.)

    +

    📗📝 Here's test_simple_login.py, which uses BaseCase class inheritance, and runs with pytest or pynose. (Use self.driver to access Selenium's raw driver.)

    ```python from seleniumbase import BaseCase @@ -187,22 +226,7 @@ class TestSimpleLogin(BaseCase): self.assert_text("signed out", "#top_message") ``` -

    📗📝 Here's a test from sb_fixture_tests.py, which uses the sb pytest fixture. Runs with pytest. (Use sb.driver to access Selenium's raw driver.)

    - -```python -def test_sb_fixture_with_no_class(sb): - sb.open("seleniumbase.io/simple/login") - sb.type("#username", "demo_user") - sb.type("#password", "secret_pass") - sb.click('a:contains("Sign in")') - sb.assert_exact_text("Welcome!", "h1") - sb.assert_element("img#image1") - sb.highlight("#image1") - sb.click_link("Sign out") - sb.assert_text("signed out", "#top_message") -``` - -

    📙📝 Here's raw_login_sb.py, which uses the SB Context Manager. Runs with pure python. (Use sb.driver to access Selenium's raw driver.)

    +

    📘📝 Here's raw_login_sb.py, which uses the SB Context Manager. Runs with pure python. (Use sb.driver to access Selenium's raw driver.)

    ```python from seleniumbase import SB @@ -219,24 +243,7 @@ with SB() as sb: sb.assert_text("signed out", "#top_message") ``` -

    📔📝 Here's raw_login_context.py, which uses the DriverContext Manager. Runs with pure python. (The driver is an improved version of Selenium's raw driver, with more methods.)

    - -```python -from seleniumbase import DriverContext - -with DriverContext() as driver: - driver.open("seleniumbase.io/simple/login") - driver.type("#username", "demo_user") - driver.type("#password", "secret_pass") - driver.click('a:contains("Sign in")') - driver.assert_exact_text("Welcome!", "h1") - driver.assert_element("img#image1") - driver.highlight("#image1") - driver.click_link("Sign out") - driver.assert_text("signed out", "#top_message") -``` - -

    📔📝 Here's raw_login_driver.py, which uses the Driver Manager. Runs with pure python. (The driver is an improved version of Selenium's raw driver, with more methods.)

    +

    📙📝 Here's raw_login_driver.py, which uses the Driver Manager. Runs with pure python. (The driver is an improved version of Selenium's raw driver, with more methods.)

    ```python from seleniumbase import Driver @@ -256,23 +263,6 @@ finally: driver.quit() ``` -

    📕📝 Here's login_app.feature, which uses behave-BDD Gherkin syntax. Runs with behave. (Learn about the SeleniumBase behave-BDD integration)

    - -```gherkin -Feature: SeleniumBase scenarios for the Simple App - - Scenario: Verify the Simple App (Login / Logout) - Given Open "seleniumbase.io/simple/login" - And Type "demo_user" into "#username" - And Type "secret_pass" into "#password" - And Click 'a:contains("Sign in")' - And Assert exact text "Welcome!" in "h1" - And Assert element "img#image1" - And Highlight "#image1" - And Click link "Sign out" - And Assert text "signed out" in "#top_message" -``` - -------- @@ -371,20 +361,21 @@ pip install -e . â–ļī¸ Here's sample output from a chromedriver download. (click to expand) ```bash -*** chromedriver to download = 121.0.6167.85 (Latest Stable) +*** chromedriver to download = 131.0.6778.108 (Latest Stable) Downloading chromedriver-mac-arm64.zip from: -https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/mac-arm64/chromedriver-mac-arm64.zip ... +https://storage.googleapis.com/chrome-for-testing-public/131.0.6778.108/mac-arm64/chromedriver-mac-arm64.zip ... Download Complete! Extracting ['chromedriver'] from chromedriver-mac-arm64.zip ... Unzip Complete! The file [chromedriver] was saved to: -/Users/michael/github/SeleniumBase/seleniumbase/drivers/chromedriver +~/github/SeleniumBase/seleniumbase/drivers/ +chromedriver -Making [chromedriver 121.0.6167.85] executable ... -[chromedriver 121.0.6167.85] is now ready for use! +Making [chromedriver 131.0.6778.108] executable ... +[chromedriver 131.0.6778.108] is now ready for use! ``` @@ -404,7 +395,7 @@ pytest my_first_test.py SeleniumBase Test -

    Here's the code for my_first_test.py:

    +

    Here's the full code for my_first_test.py:

    ```python from seleniumbase import BaseCase From bf0cfbdca5cabf5652eacf3b68073e9dc31582c1 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Sun, 15 Dec 2024 15:28:43 -0500 Subject: [PATCH 6/7] Refresh Python dependencies --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8f3f8fed029..a52ab7e25da 100755 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ setuptools~=70.2;python_version<"3.10" setuptools>=75.6.0;python_version>="3.10" wheel>=0.45.1 attrs>=24.2.0 -certifi>=2024.8.30 +certifi>=2024.12.14 exceptiongroup>=1.2.2 websockets~=13.1;python_version<"3.9" websockets>=14.1;python_version>="3.9" diff --git a/setup.py b/setup.py index 9ae77f4b8fb..8a50ea512c7 100755 --- a/setup.py +++ b/setup.py @@ -153,7 +153,7 @@ 'setuptools>=75.6.0;python_version>="3.10"', 'wheel>=0.45.1', 'attrs>=24.2.0', - "certifi>=2024.8.30", + "certifi>=2024.12.14", "exceptiongroup>=1.2.2", 'websockets~=13.1;python_version<"3.9"', 'websockets>=14.1;python_version>="3.9"', From b9ebe7028e4f8d59c2a417607736962bdad71d4a Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Sun, 15 Dec 2024 15:29:13 -0500 Subject: [PATCH 7/7] Version 4.33.11 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index dc9b0326b88..46f03196f5e 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.33.10" +__version__ = "4.33.11"