From 8414903e0035674af4a237e7654022886f138f3a Mon Sep 17 00:00:00 2001 From: Henning Merklinger Date: Wed, 25 Jan 2023 13:11:24 +0100 Subject: [PATCH 1/5] added caching for datadome cookie --- src/models/config.py | 10 +++++++++- src/scanner.py | 3 +++ src/tgtg/tgtg_client.py | 12 ++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/models/config.py b/src/models/config.py index a3c6f567..44b3de58 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -24,6 +24,7 @@ 'access_token': None, 'refresh_token': None, 'user_id': None, + 'datadome': None, 'timeout': 60, 'access_token_lifetime': 14400, 'max_polling_tries': 24, @@ -138,6 +139,8 @@ def _load_tokens(self) -> None: self.tgtg["refresh_token"] = file.read() with self._open('userID', 'r') as file: self.tgtg["user_id"] = file.read() + with self._open('datadome', 'r') as file: + self.tgtg["datadome"] = file.read() except FileNotFoundError: log.warning("No token files in token path.") except EnvironmentError as err: @@ -212,6 +215,7 @@ def _read_ini(self) -> None: self._ini_get(config, "TGTG", "AccessToken", "tgtg.access_token") self._ini_get(config, "TGTG", "RefreshToken", "tgtg.refresh_token") self._ini_get(config, "TGTG", "UserId", "tgtg.user_id") + self._ini_get(config, "TGTG", "Datadome", "tgtg.datadome") self._ini_get_int(config, "TGTG", "Timeout", "tgtg.timeout") self._ini_get_int(config, "TGTG", "AccessTokenLifetime", "tgtg.access_token_lifetime") @@ -310,6 +314,7 @@ def _read_env(self) -> None: self._env_get("TGTG_ACCESS_TOKEN", "tgtg.access_token") self._env_get("TGTG_REFRESH_TOKEN", "tgtg.refresh_token") self._env_get("TGTG_USER_ID", "tgtg.user_id") + self._env_get("TGTG_DATADOME", "tgtg.datadome") self._env_get_int("TGTG_TIMEOUT", "tgtg.timeout") self._env_get_int("TGTG_ACCESS_TOKEN_LIFETIME", "tgtg.access_token_lifetime") @@ -380,7 +385,7 @@ def set(self, section: str, option: str, value: Any) -> bool: return False def save_tokens(self, access_token: str, refresh_token: str, - user_id: str) -> None: + user_id: str, datadome: str) -> None: """ Saves TGTG Access Tokens to config.ini if provided or as files to token_path. @@ -395,6 +400,7 @@ def save_tokens(self, access_token: str, refresh_token: str, config.set("TGTG", "AccessToken", access_token) config.set("TGTG", "RefreshToken", refresh_token) config.set("TGTG", "UserId", user_id) + config.set("TGTG", "Datadome", datadome) with open(self.file, 'w', encoding='utf-8') as configfile: config.write(configfile) except EnvironmentError as err: @@ -407,5 +413,7 @@ def save_tokens(self, access_token: str, refresh_token: str, file.write(refresh_token) with self._open('userID', 'w') as file: file.write(user_id) + with self._open('datadome', 'w') as file: + file.write(datadome) except EnvironmentError as err: log.error("error saving credentials! - %s", err) diff --git a/src/scanner.py b/src/scanner.py index e59e45b5..9e3ae178 100644 --- a/src/scanner.py +++ b/src/scanner.py @@ -29,6 +29,7 @@ def __init__(self, config: Config): access_token=self.config.tgtg.get("access_token"), refresh_token=self.config.tgtg.get("refresh_token"), user_id=self.config.tgtg.get("user_id"), + datadome_cookie=self.config.tgtg.get("datadome") ) def _get_test_item(self) -> Item: @@ -77,6 +78,7 @@ def _job(self) -> None: self.tgtg_client.access_token, self.tgtg_client.refresh_token, self.tgtg_client.user_id, + self.tgtg_client.datadome_cookie ) def _get_favorites(self) -> list[Item]: @@ -156,6 +158,7 @@ def run(self) -> NoReturn: self.tgtg_client.access_token, self.tgtg_client.refresh_token, self.tgtg_client.user_id, + self.tgtg_client.datadome_cookie ) # start scanner log.info("Scanner started ...") diff --git a/src/tgtg/tgtg_client.py b/src/tgtg/tgtg_client.py index a55414a0..bdd3868f 100644 --- a/src/tgtg/tgtg_client.py +++ b/src/tgtg/tgtg_client.py @@ -57,7 +57,7 @@ class TgtgSession(requests.Session): def __init__(self, user_agent: str = None, language: str = "en-UK", timeout: int = None, proxies: dict = None, - *args, **kwargs) -> None: + datadome_cookie: str = None, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.mount("https://", self.http_adapter) self.mount("http://", self.http_adapter) @@ -68,6 +68,10 @@ def __init__(self, user_agent: str = None, language: str = "en-UK", } self.timeout = timeout self.proxies = proxies + if datadome_cookie: + self.cookies.set("datadome", datadome_cookie, + domain=".apptoogoodtogo.com", + path="/", secure=True) def post(self, url: str, access_token: str = None, **kwargs ) -> requests.Response: @@ -94,6 +98,7 @@ def __init__( access_token=None, refresh_token=None, user_id=None, + datadome_cookie=None, user_agent=None, language="en-UK", proxies=None, @@ -109,6 +114,7 @@ def __init__( self.access_token = access_token self.refresh_token = refresh_token self.user_id = user_id + self.datadome_cookie = datadome_cookie self.last_time_token_refreshed = None self.access_token_lifetime = access_token_lifetime @@ -138,7 +144,8 @@ def _create_session(self) -> TgtgSession: return TgtgSession(self.user_agent, self.language, self.timeout, - self.proxies) + self.proxies, + self.datadome_cookie) def get_credentials(self) -> dict: """Returns current tgtg api credentials. @@ -162,6 +169,7 @@ def _post(self, path, **kwargs) -> requests.Response: access_token=self.access_token, **kwargs, ) + self.datadome_cookie = self.session.cookies.get("datadome") if response.status_code in (HTTPStatus.OK, HTTPStatus.ACCEPTED): self.captcha_error_count = 0 return response From 2674ef683e080021a844772ee9a1a6596a0cf535 Mon Sep 17 00:00:00 2001 From: Henning Merklinger Date: Wed, 25 Jan 2023 13:16:57 +0100 Subject: [PATCH 2/5] updated tests for datadome cookie --- tests/test_config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index e449c798..6e50ce1a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -38,8 +38,8 @@ def test_save_tokens_to_ini(temp_path: Path): config_path = Path(temp_path, "config.ini") config_path.touch(exist_ok=True) config = Config(config_path.absolute()) - config.save_tokens("test_access_token", - "test_refresh_token", "test_user_id") + config.save_tokens("test_access_token", "test_refresh_token", + "test_user_id", "test_cookie") config_parser = configparser.ConfigParser() config_parser.read(config_path, encoding='utf-8') @@ -47,16 +47,18 @@ def test_save_tokens_to_ini(temp_path: Path): assert config_parser.get("TGTG", "AccessToken") == "test_access_token" assert config_parser.get("TGTG", "RefreshToken") == "test_refresh_token" assert config_parser.get("TGTG", "UserId") == "test_user_id" + assert config_parser.get("TGTG", "Datadome") == "test_cookie" def test_token_path(temp_path: Path, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("TGTG_TOKEN_PATH", str(temp_path.absolute())) config = Config() - config.save_tokens("test_access_token", - "test_refresh_token", "test_user_id") + config.save_tokens("test_access_token", "test_refresh_token", + "test_user_id", "test_cookie") config._load_tokens() assert config.tgtg.get("access_token") == "test_access_token" assert config.tgtg.get("refresh_token") == "test_refresh_token" assert config.tgtg.get("user_id") == "test_user_id" + assert config.tgtg.get("datadome") == "test_cookie" From 391c983b6e080f1715fba676a7c898df3005026b Mon Sep 17 00:00:00 2001 From: Henning Merklinger Date: Wed, 25 Jan 2023 20:08:12 +0100 Subject: [PATCH 3/5] remove datadome cookie after 3. retry --- src/tgtg/tgtg_client.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/tgtg/tgtg_client.py b/src/tgtg/tgtg_client.py index bdd3868f..a54e6a43 100644 --- a/src/tgtg/tgtg_client.py +++ b/src/tgtg/tgtg_client.py @@ -178,23 +178,26 @@ def _post(self, path, **kwargs) -> requests.Response: except ValueError: # Status Code == 403 and no json contend # --> Blocked due to rate limit / wrong user_agent. - # 1. Get latest APK Version from google - # 2. Reset current session - # 3. Sleep 10 seconds, after 10 errors sleep 1 hour - # 4. Rety request + # 1. Try: Get latest APK Version from google + # 2. Try: Reset session + # 3. Try: Delete datadome cookie and reset session + # 10.Try: Sleep 10 minutes, and reset session if response.status_code == 403: log.debug("Captcha Error 403!") + self.captcha_error_count += 1 if self.captcha_error_count == 1: self.user_agent = self._get_user_agent() + elif self.captcha_error_count == 2: self.session = self._create_session() - self.captcha_error_count += 1 - if self.captcha_error_count > 10: + elif self.captcha_error_count == 4: + self.datadome_cookie = None + self.session = self._create_session() + elif self.captcha_error_count >= 10: log.warning( "Too many captcha Errors! Sleeping for 10 minutes...") time.sleep(10 * 60) log.info("Retrying ...") self.captcha_error_count = 0 - self.user_agent = self._get_user_agent() self.session = self._create_session() time.sleep(1) return self._post(path, **kwargs) From c9c1ca1018fbae7e1a4f0076a32ae1c4f723d66f Mon Sep 17 00:00:00 2001 From: Henning Merklinger Date: Fri, 27 Jan 2023 18:53:46 +0100 Subject: [PATCH 4/5] update dependencies --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6dda746c..81ae841a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,18 @@ APScheduler==3.6.3 cachetools==4.2.2 certifi==2022.12.7 -charset-normalizer==2.1.1 +charset-normalizer==3.0.1 colorlog==6.7.0 -cron-descriptor==1.2.32 +cron-descriptor==1.2.35 idna==3.4 packaging==23.0 -prometheus-client==0.15.0 +prometheus-client==0.16.0 pycron==3.0.0 python-pushsafer==1.0 python-telegram-bot==13.15 -pytz==2022.7 +pytz==2022.7.1 pytz-deprecation-shim==0.1.0.post0 -requests==2.28.1 +requests==2.28.2 six==1.16.0 tornado==6.1 tzdata==2022.7 From 69e879eb0f77abaf6fc2eff26e447a8c5b73e388 Mon Sep 17 00:00:00 2001 From: Henning Merklinger Date: Fri, 27 Jan 2023 18:54:51 +0100 Subject: [PATCH 5/5] bump version 1.14.8 --- DOCKER_README.md | 4 ++-- src/_version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCKER_README.md b/DOCKER_README.md index c15937fd..cbd7d5d1 100644 --- a/DOCKER_README.md +++ b/DOCKER_README.md @@ -10,8 +10,8 @@ Readme, source and documentation on [https://github.com/Der-Henning/tgtg](https: - [`edge`](https://github.com/Der-Henning/tgtg/blob/main/Dockerfile) - [`edge-alpine`](https://github.com/Der-Henning/tgtg/blob/main/Dockerfile.alpine) -- [`v1`, `v1.14`, `v1.14.7`, `latest`](https://github.com/Der-Henning/tgtg/blob/v1.14.7/Dockerfile) -- [`v1-alpine`, `v1.14-alpine`, `v1.14.7-alpine`, `latest-alpine`](https://github.com/Der-Henning/tgtg/blob/v1.14.7/Dockerfile.alpine) +- [`v1`, `v1.14`, `v1.14.8`, `latest`](https://github.com/Der-Henning/tgtg/blob/v1.14.8/Dockerfile) +- [`v1-alpine`, `v1.14-alpine`, `v1.14.8-alpine`, `latest-alpine`](https://github.com/Der-Henning/tgtg/blob/v1.14.8/Dockerfile.alpine) # Quick Start diff --git a/src/_version.py b/src/_version.py index aa020e8f..d4af8483 100644 --- a/src/_version.py +++ b/src/_version.py @@ -1,6 +1,6 @@ __title__ = "TGTG Scanner" __description__ = "Provides notifications for TGTG magic bags" -__version__ = "1.14.7" +__version__ = "1.14.8" __author__ = "Henning Merklinger" __author_email__ = "henning.merklinger@gmail.com" __license__ = "GPL"