diff --git a/scripts/CI/CI_LETS_DO_THIS.py b/scripts/CI/CI_LETS_DO_THIS.py index e47be58..8f69c1f 100755 --- a/scripts/CI/CI_LETS_DO_THIS.py +++ b/scripts/CI/CI_LETS_DO_THIS.py @@ -40,6 +40,8 @@ async def lets_go(session: aiohttp.ClientSession, url, limit): print(e) return file_name +dns_first = "9.9.9.10" +dns_second = "149.112.112.10" async def bulk_lets_go(limit_value, urls): limit = asyncio.Semaphore(limit_value) @@ -48,7 +50,7 @@ async def bulk_lets_go(limit_value, urls): for result in results: if result: ECO_result = subprocess.run([pj(main_path, "scripts", "ECODFF.py"), pj( - main_path, result), "-c 40"], check=False, capture_output=True, text=True) + main_path, result), "-c 40", "--dns", dns_first, dns_second], check=False, capture_output=True, text=True) if ECO_error := ECO_result.stderr: print(ECO_error) if ECO_output := ECO_result.stdout: @@ -136,7 +138,7 @@ async def download(file_name, url): print(s_result.stdout) elif sys.argv[1].startswith("KAD_") or sys.argv[1].startswith("KADhosts_"): ECO_result = subprocess.run([pj(main_path, "scripts", "ECODFF.py"), pj( - main_path, "split", sys.argv[1]), "-c 40"], check=False, capture_output=True, text=True) + main_path, "split", sys.argv[1]), "-c 40", "--dns", dns_first, dns_second], check=False, capture_output=True, text=True) if ECO_error := ECO_result.stderr: print(ECO_error) if ECO_output := ECO_result.stdout: diff --git a/scripts/ECODFF.py b/scripts/ECODFF.py index dab541f..e566249 100755 --- a/scripts/ECODFF.py +++ b/scripts/ECODFF.py @@ -42,7 +42,7 @@ import git # Version number -SCRIPT_VERSION = "2.0.34" +SCRIPT_VERSION = "2.0.35" # Parse arguments parser = argparse.ArgumentParser() @@ -50,6 +50,7 @@ parser.add_argument("-c", "--connections", type=int, action='store', default=20) parser.add_argument("-v", "--version", action='version', version="ECODFF" + ' ' + SCRIPT_VERSION) +parser.add_argument("--dns", action='store', type=str, nargs="+") args = parser.parse_args() pj = os.path.join @@ -76,7 +77,9 @@ with open(pj(EXPIRED_DIR, ".keep"), 'w', encoding="utf-8") as fp: pass -DNS_a = ["9.9.9.10", "149.112.112.10"] +DNS_a = "" +if args.dns: + DNS_a = args.dns for path_to_file in args.path_to_file: FILTERLIST = os.path.splitext(os.path.basename(path_to_file))[0] @@ -134,7 +137,8 @@ sem_value = args.connections custom_resolver = dns.asyncresolver.Resolver() - custom_resolver.nameservers = DNS_a + if DNS_a: + custom_resolver.nameservers = DNS_a SUB_PAT = re.compile(r"(.+\.)+.+\..+$") @@ -143,6 +147,7 @@ async def domain_dns_check(domain, limit): status = "online" try: print(f"Checking the status of {domain}...") + await asyncio.sleep(1) answers_NS = await custom_resolver.resolve(domain, "NS") except NXDOMAIN: status = "offline" @@ -160,7 +165,6 @@ async def domain_dns_check(domain, limit): status = "parked" finally: result = f"{domain} {status}" - await asyncio.sleep(1) return result async def bulk_domain_dns_check(limit_value): @@ -294,6 +298,7 @@ async def get_status_code(session: aiohttp.ClientSession, url: str, limit): async with limit: try: print(f"Checking the status of {url}...") + await asyncio.sleep(1) resp = await session.get(f"http://{url}", allow_redirects=False) status_code = resp.status if (400 <= int(status_code) <= 499) and SUB_PAT.search(url): @@ -353,14 +358,15 @@ async def get_status_code(session: aiohttp.ClientSession, url: str, limit): result = "" if "status_code" in locals(): result = f"{str(url)} {str(status_code)}" - await asyncio.sleep(1) return result async def save_status_code(timeout_time, limit_value): session_timeout = aiohttp.ClientTimeout( total=None, sock_connect=timeout_time, sock_read=timeout_time) limit = asyncio.Semaphore(limit_value) - resolver = aiohttp.AsyncResolver(nameservers=DNS_a) + resolver = aiohttp.AsyncResolver() + if DNS_a: + resolver = aiohttp.AsyncResolver(nameservers=DNS_a) async with aiohttp.ClientSession(timeout=session_timeout, connector=aiohttp.TCPConnector(resolver=resolver), headers=request_headers) as session: statuses = await asyncio.gather(*[get_status_code(session, url, limit) for url in unknown_pages]) with open(UNKNOWN_FILE, 'w', encoding="utf-8") as u_f: