Merge pull request #255 from Der-Henning/dev

Cache datadome cookie to reduce captcha errors
Der-Henning · Jan 27, 2023 · 89c8cd7 · 89c8cd7
2 parents 584613f + 69e879e
commit 89c8cd7
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 22 deletions.
diff --git a/DOCKER_README.md b/DOCKER_README.md
@@ -10,8 +10,8 @@ Readme, source and documentation on [https://github.com/Der-Henning/tgtg](https:
 
 - [`edge`](https://github.com/Der-Henning/tgtg/blob/main/Dockerfile)
 - [`edge-alpine`](https://github.com/Der-Henning/tgtg/blob/main/Dockerfile.alpine)
-- [`v1`, `v1.14`, `v1.14.7`, `latest`](https://github.com/Der-Henning/tgtg/blob/v1.14.7/Dockerfile)
-- [`v1-alpine`, `v1.14-alpine`, `v1.14.7-alpine`, `latest-alpine`](https://github.com/Der-Henning/tgtg/blob/v1.14.7/Dockerfile.alpine)
+- [`v1`, `v1.14`, `v1.14.8`, `latest`](https://github.com/Der-Henning/tgtg/blob/v1.14.8/Dockerfile)
+- [`v1-alpine`, `v1.14-alpine`, `v1.14.8-alpine`, `latest-alpine`](https://github.com/Der-Henning/tgtg/blob/v1.14.8/Dockerfile.alpine)
 
 # Quick Start
 

diff --git a/requirements.txt b/requirements.txt
@@ -1,18 +1,18 @@
 APScheduler==3.6.3
 cachetools==4.2.2
 certifi==2022.12.7
-charset-normalizer==2.1.1
+charset-normalizer==3.0.1
 colorlog==6.7.0
-cron-descriptor==1.2.32
+cron-descriptor==1.2.35
 idna==3.4
 packaging==23.0
-prometheus-client==0.15.0
+prometheus-client==0.16.0
 pycron==3.0.0
 python-pushsafer==1.0
 python-telegram-bot==13.15
-pytz==2022.7
+pytz==2022.7.1
 pytz-deprecation-shim==0.1.0.post0
-requests==2.28.1
+requests==2.28.2
 six==1.16.0
 tornado==6.1
 tzdata==2022.7

diff --git a/src/_version.py b/src/_version.py
@@ -1,6 +1,6 @@
 __title__ = "TGTG Scanner"
 __description__ = "Provides notifications for TGTG magic bags"
-__version__ = "1.14.7"
+__version__ = "1.14.8"
 __author__ = "Henning Merklinger"
 __author_email__ = "[email protected]"
 __license__ = "GPL"

diff --git a/src/models/config.py b/src/models/config.py
@@ -24,6 +24,7 @@
         'access_token': None,
         'refresh_token': None,
         'user_id': None,
+        'datadome': None,
         'timeout': 60,
         'access_token_lifetime': 14400,
         'max_polling_tries': 24,
@@ -138,6 +139,8 @@ def _load_tokens(self) -> None:
                     self.tgtg["refresh_token"] = file.read()
                 with self._open('userID', 'r') as file:
                     self.tgtg["user_id"] = file.read()
+                with self._open('datadome', 'r') as file:
+                    self.tgtg["datadome"] = file.read()
             except FileNotFoundError:
                 log.warning("No token files in token path.")
             except EnvironmentError as err:
@@ -212,6 +215,7 @@ def _read_ini(self) -> None:
             self._ini_get(config, "TGTG", "AccessToken", "tgtg.access_token")
             self._ini_get(config, "TGTG", "RefreshToken", "tgtg.refresh_token")
             self._ini_get(config, "TGTG", "UserId", "tgtg.user_id")
+            self._ini_get(config, "TGTG", "Datadome", "tgtg.datadome")
             self._ini_get_int(config, "TGTG", "Timeout", "tgtg.timeout")
             self._ini_get_int(config, "TGTG", "AccessTokenLifetime",
                               "tgtg.access_token_lifetime")
@@ -310,6 +314,7 @@ def _read_env(self) -> None:
             self._env_get("TGTG_ACCESS_TOKEN", "tgtg.access_token")
             self._env_get("TGTG_REFRESH_TOKEN", "tgtg.refresh_token")
             self._env_get("TGTG_USER_ID", "tgtg.user_id")
+            self._env_get("TGTG_DATADOME", "tgtg.datadome")
             self._env_get_int("TGTG_TIMEOUT", "tgtg.timeout")
             self._env_get_int("TGTG_ACCESS_TOKEN_LIFETIME",
                               "tgtg.access_token_lifetime")
@@ -380,7 +385,7 @@ def set(self, section: str, option: str, value: Any) -> bool:
         return False
 
     def save_tokens(self, access_token: str, refresh_token: str,
-                    user_id: str) -> None:
+                    user_id: str, datadome: str) -> None:
         """
         Saves TGTG Access Tokens to config.ini
         if provided or as files to token_path.
@@ -395,6 +400,7 @@ def save_tokens(self, access_token: str, refresh_token: str,
                 config.set("TGTG", "AccessToken", access_token)
                 config.set("TGTG", "RefreshToken", refresh_token)
                 config.set("TGTG", "UserId", user_id)
+                config.set("TGTG", "Datadome", datadome)
                 with open(self.file, 'w', encoding='utf-8') as configfile:
                     config.write(configfile)
             except EnvironmentError as err:
@@ -407,5 +413,7 @@ def save_tokens(self, access_token: str, refresh_token: str,
                     file.write(refresh_token)
                 with self._open('userID', 'w') as file:
                     file.write(user_id)
+                with self._open('datadome', 'w') as file:
+                    file.write(datadome)
             except EnvironmentError as err:
                 log.error("error saving credentials! - %s", err)
diff --git a/src/scanner.py b/src/scanner.py
@@ -29,6 +29,7 @@ def __init__(self, config: Config):
             access_token=self.config.tgtg.get("access_token"),
             refresh_token=self.config.tgtg.get("refresh_token"),
             user_id=self.config.tgtg.get("user_id"),
+            datadome_cookie=self.config.tgtg.get("datadome")
         )
 
     def _get_test_item(self) -> Item:
@@ -77,6 +78,7 @@ def _job(self) -> None:
             self.tgtg_client.access_token,
             self.tgtg_client.refresh_token,
             self.tgtg_client.user_id,
+            self.tgtg_client.datadome_cookie
         )
 
     def _get_favorites(self) -> list[Item]:
@@ -156,6 +158,7 @@ def run(self) -> NoReturn:
             self.tgtg_client.access_token,
             self.tgtg_client.refresh_token,
             self.tgtg_client.user_id,
+            self.tgtg_client.datadome_cookie
         )
         # start scanner
         log.info("Scanner started ...")

diff --git a/src/tgtg/tgtg_client.py b/src/tgtg/tgtg_client.py
@@ -57,7 +57,7 @@ class TgtgSession(requests.Session):
 
     def __init__(self, user_agent: str = None, language: str = "en-UK",
                  timeout: int = None, proxies: dict = None,
-                 *args, **kwargs) -> None:
+                 datadome_cookie: str = None, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.mount("https://", self.http_adapter)
         self.mount("http://", self.http_adapter)
@@ -68,6 +68,10 @@ def __init__(self, user_agent: str = None, language: str = "en-UK",
         }
         self.timeout = timeout
         self.proxies = proxies
+        if datadome_cookie:
+            self.cookies.set("datadome", datadome_cookie,
+                             domain=".apptoogoodtogo.com",
+                             path="/", secure=True)
 
     def post(self, url: str, access_token: str = None, **kwargs
              ) -> requests.Response:
@@ -94,6 +98,7 @@ def __init__(
         access_token=None,
         refresh_token=None,
         user_id=None,
+        datadome_cookie=None,
         user_agent=None,
         language="en-UK",
         proxies=None,
@@ -109,6 +114,7 @@ def __init__(
         self.access_token = access_token
         self.refresh_token = refresh_token
         self.user_id = user_id
+        self.datadome_cookie = datadome_cookie
 
         self.last_time_token_refreshed = None
         self.access_token_lifetime = access_token_lifetime
@@ -138,7 +144,8 @@ def _create_session(self) -> TgtgSession:
         return TgtgSession(self.user_agent,
                            self.language,
                            self.timeout,
-                           self.proxies)
+                           self.proxies,
+                           self.datadome_cookie)
 
     def get_credentials(self) -> dict:
         """Returns current tgtg api credentials.
@@ -162,6 +169,7 @@ def _post(self, path, **kwargs) -> requests.Response:
             access_token=self.access_token,
             **kwargs,
         )
+        self.datadome_cookie = self.session.cookies.get("datadome")
         if response.status_code in (HTTPStatus.OK, HTTPStatus.ACCEPTED):
             self.captcha_error_count = 0
             return response
@@ -170,23 +178,26 @@ def _post(self, path, **kwargs) -> requests.Response:
         except ValueError:
             # Status Code == 403 and no json contend
             # --> Blocked due to rate limit / wrong user_agent.
-            # 1. Get latest APK Version from google
-            # 2. Reset current session
-            # 3. Sleep 10 seconds, after 10 errors sleep 1 hour
-            # 4. Rety request
+            # 1. Try: Get latest APK Version from google
+            # 2. Try: Reset session
+            # 3. Try: Delete datadome cookie and reset session
+            # 10.Try: Sleep 10 minutes, and reset session
             if response.status_code == 403:
                 log.debug("Captcha Error 403!")
+                self.captcha_error_count += 1
                 if self.captcha_error_count == 1:
                     self.user_agent = self._get_user_agent()
+                elif self.captcha_error_count == 2:
                     self.session = self._create_session()
-                self.captcha_error_count += 1
-                if self.captcha_error_count > 10:
+                elif self.captcha_error_count == 4:
+                    self.datadome_cookie = None
+                    self.session = self._create_session()
+                elif self.captcha_error_count >= 10:
                     log.warning(
                         "Too many captcha Errors! Sleeping for 10 minutes...")
                     time.sleep(10 * 60)
                     log.info("Retrying ...")
                     self.captcha_error_count = 0
-                    self.user_agent = self._get_user_agent()
                     self.session = self._create_session()
                 time.sleep(1)
                 return self._post(path, **kwargs)

diff --git a/tests/test_config.py b/tests/test_config.py
@@ -38,25 +38,27 @@ def test_save_tokens_to_ini(temp_path: Path):
     config_path = Path(temp_path, "config.ini")
     config_path.touch(exist_ok=True)
     config = Config(config_path.absolute())
-    config.save_tokens("test_access_token",
-                       "test_refresh_token", "test_user_id")
+    config.save_tokens("test_access_token", "test_refresh_token",
+                       "test_user_id", "test_cookie")
 
     config_parser = configparser.ConfigParser()
     config_parser.read(config_path, encoding='utf-8')
 
     assert config_parser.get("TGTG", "AccessToken") == "test_access_token"
     assert config_parser.get("TGTG", "RefreshToken") == "test_refresh_token"
     assert config_parser.get("TGTG", "UserId") == "test_user_id"
+    assert config_parser.get("TGTG", "Datadome") == "test_cookie"
 
 
 def test_token_path(temp_path: Path, monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setenv("TGTG_TOKEN_PATH", str(temp_path.absolute()))
 
     config = Config()
-    config.save_tokens("test_access_token",
-                       "test_refresh_token", "test_user_id")
+    config.save_tokens("test_access_token", "test_refresh_token",
+                       "test_user_id", "test_cookie")
     config._load_tokens()
 
     assert config.tgtg.get("access_token") == "test_access_token"
     assert config.tgtg.get("refresh_token") == "test_refresh_token"
     assert config.tgtg.get("user_id") == "test_user_id"
+    assert config.tgtg.get("datadome") == "test_cookie"