diff --git a/scripts/Dockerfile.pypy3 b/scripts/Dockerfile.pypy3 new file mode 100644 index 0000000..801cf63 --- /dev/null +++ b/scripts/Dockerfile.pypy3 @@ -0,0 +1,8 @@ +FROM pypy:3-bookworm + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y libgeos-dev + +COPY requirements.txt / +RUN pip install -r requirements.txt diff --git a/scripts/runall.py b/scripts/runall.py index ada2de8..63bf37e 100755 --- a/scripts/runall.py +++ b/scripts/runall.py @@ -13,6 +13,7 @@ from operator import itemgetter from pathlib import Path from zlib import crc32 +import sqlite3 RED = "\033[91m" GREEN = "\033[92m" @@ -38,108 +39,122 @@ "C++": "{year}/build/day{day}_cpp", } +INTERPRETERS = { + "Python": "python3", + "PyPy": "pypy3", +} + def get_cache(): """Retrieve the cache instance from memory or load it from disk.""" cache = globals().get("_cache") if cache is None: - cache_file = Path(__file__).parent.parent / "data" / "cache.json" - if cache_file.is_file(): - cache = json.loads(cache_file.read_bytes()) - else: - cache = {} + cache_file = Path(__file__).parent.parent / "data" / "cache.db" + + cache_db = sqlite3.connect(cache_file) + + cache_db.executescript( + "create table if not exists solutions (" + " key text primary key not null," + " mtime_ns int," + " elapsed float," + " status text," + " answers text);" + ) + cache = {"db": cache_db, "modified": False} globals()["_cache"] = cache - cache["modified"] = False + return cache def save_cache(): - cache = get_cache() - if cache["modified"]: - cache.pop("modified") - cache_file = Path(__file__).parent.parent / "data" / "cache.json" - cache_file.write_text(json.dumps(cache, indent=2, ensure_ascii=True)) - cache["modified"] = False - print(f"{FEINT}{ITALIC}cache commited{RESET}") + pass + # cache = get_cache() + # if cache["modified"]: + # cache["db"].commit() + # cache["modified"] = False + # print(f"{FEINT}{ITALIC}cache commited{RESET}") -def check_cache(key, file_timestamp: Path, no_check=False): +def check_cache(key, file_timestamp: Path, no_age_check=False): cache = get_cache() key = str(key) - e = cache.get(key, None) - if e: + db = cache["db"] + cursor = db.execute("select mtime_ns,elapsed,status,answers from solutions where key=?", (key,)) + row = cursor.fetchone() + if row: timestamp = file_timestamp.stat().st_mtime_ns - if e["timestamp"] == timestamp or no_check: - return e + if row[0] == timestamp or no_age_check: + return { + "elapsed": row[1], + "status": row[2], + "answers": row[3].split("\n"), + } else: - seconds = round((timestamp - e["timestamp"]) / 1000000000) - delta = timedelta(seconds=seconds) + # seconds = round((timestamp - e["timestamp"]) / 1000000000) + # delta = timedelta(seconds=seconds) + # print(f"{FEINT}{ITALIC}entry {key} is out of date for {delta}{RESET}", end=f"{CR}") - print(f"{FEINT}{ITALIC}entry {key} is out of date for {delta}{RESET}", end=f"{CR}") + print(f"{FEINT}{ITALIC}entry {key} is out of date{RESET}", end=f"{CR}") else: print(f"{FEINT}{ITALIC}missing cache for {key}{RESET}", end=f"{CR}") -def update_cache(key, timestamp: Path, elapsed, status, answers): +def update_cache(key, timestamp: Path, elapsed: float, status: str, answers: t.Iterable): cache = get_cache() + db = cache["db"] key = str(key) - e = cache.get(key, {}) - e["timestamp"] = timestamp.stat().st_mtime_ns - e["elapsed"] = elapsed - e["status"] = status - e["answers"] = answers - cache[key] = e - cache["modified"] = True - return e + db.execute( + "insert or replace into solutions (key,mtime_ns,elapsed,status,answers) values (?,?,?,?,?)", + (key, timestamp.stat().st_mtime_ns, elapsed, status, "\n".join(answers)), + ) + + # cache["modified"] = True + db.commit() -def run(key: str, prog: Path, lang: str, file: Path, solution: t.List, refresh: bool, dry_run: bool): + return { + "elapsed": elapsed, + "status": status, + "answers": answers, + } + + +def run(prog: Path, lang: str, file: Path, solution: t.List, warmup: bool) -> t.Dict[str, t.Any]: if not prog.is_file(): return cmd = [prog.absolute().as_posix()] - if lang == "Python": - cmd.insert(0, "python3") - elif lang == "PyPy": - cmd.insert(0, "pypy3") - - if refresh: - e = None - else: - e = check_cache(key, prog, dry_run) - if dry_run and not e: - return None - if e: - in_cache = True + # add the interpreter + interpreter = INTERPRETERS.get(lang) + if interpreter: + cmd.insert(0, interpreter) + + if warmup and lang == "Rust": + # under macOS, the first launch of a Rust program is slower (why ???) + subprocess.call(cmd + ["--help"], stdout=subprocess.DEVNULL) + + start = time.time_ns() + out = subprocess.run(cmd + [file.absolute()], stdout=subprocess.PIPE) + elapsed = time.time_ns() - start + answers = out.stdout.decode().strip() + + status = "unknown" + if solution: + solution = solution.read_text() + if answers == solution.strip(): + status = "ok" + else: + status = "error" else: - in_cache = False - - start = time.time_ns() - out = subprocess.run(cmd + [file.absolute()], stdout=subprocess.PIPE) - elapsed = time.time_ns() - start - answers = out.stdout.decode().strip() - - status = "unknown" - if solution: - solution = solution.read_text() - if answers == solution.strip(): - status = "ok" - else: - status = "error" + if answers == "": + status = "missing" else: - if answers == "": - status = "fail" - else: - status = "unknown" - - e = update_cache(key, prog, elapsed, status, answers.split("\n")) - - e = deepcopy(e) - e["cache"] = in_cache + status = "unknown" - return e + return {"elapsed": elapsed, "status": status, "answers": answers.split("\n")} def make(year: Path, source: Path, dest: Path, cmd: str): @@ -159,7 +174,7 @@ def make(year: Path, source: Path, dest: Path, cmd: str): subprocess.check_call(cmdline, shell=True) -def build_all(filter_year): +def build_all(filter_year: int): for year in range(2015, 2024): if filter_year != 0 and year != filter_year: continue @@ -183,13 +198,20 @@ def build_all(filter_year): make(year, src, f"day{day}_cpp", "c++ -std=c++17") -def load_data(filter_year): +def load_data(filter_year, filter_user): inputs = defaultdict(dict) solutions = defaultdict(dict) for f in Path("data").rglob("*.in"): + + if f.name.startswith("._"): + continue + assert len(f.parts) == 4 + if filter_user and f.parent.parent.name != filter_user: + continue + year = int(f.parent.name) day = int(f.stem) @@ -201,7 +223,7 @@ def load_data(filter_year): crc = e["status"] else: crc = hex(crc32(f.read_bytes().strip()) & 0xFFFFFFFF) - update_cache(f, f, 0, crc, 0) + update_cache(f, f, 0, crc, []) if crc not in inputs[year, day]: inputs[year, day][crc] = f @@ -219,7 +241,7 @@ def run_day( ): elapsed = defaultdict(list) - first = True + warmup = defaultdict(lambda: True) day_suffix = mday.removeprefix(str(day)) name_max_len = 16 - len(day_suffix) @@ -242,12 +264,20 @@ def run_day( prog = Path(pattern.format(year=year, day=mday)) key = ":".join(map(str, (year, day, crc, prog, lang.lower()))) - if lang.lower() == "rust" and first and prog.is_file(): - # under macOS, the first launch of a program is slower - first = False - subprocess.call([prog, "--help"], stdout=subprocess.DEVNULL) + if refresh: + e = None + in_cache = False + else: + e = check_cache(key, prog, dry_run) + in_cache = e is not None + + if not in_cache and not dry_run: + + e = run(prog, lang, file, day_sols.get(crc), warmup[lang]) - e = run(key, prog, lang, file, day_sols.get(crc), refresh, dry_run) + if e: + warmup[lang] = False + e = update_cache(key, prog, e["elapsed"], e["status"], e["answers"]) if not e: continue @@ -257,7 +287,7 @@ def run_day( else: info = "" - status_color = {"fail": MAGENTA, "unknown": GRAY, "error": RED, "ok": GREEN}[e["status"]] + status_color = {"missing": MAGENTA, "unknown": GRAY, "error": RED, "ok": GREEN}[e["status"]] line = ( f"{CR}{RESET}{CLEAR_EOL}" @@ -265,16 +295,16 @@ def run_day( f" {YELLOW}{lang:<7}{RESET}:" f" {status_color}{e['status']:7}{RESET}" f" {WHITE}{e['elapsed']/1e9:7.3f}s" - f" {GRAY}{'☽' if e['cache'] else ' '}" + f" {GRAY}{'☽' if in_cache else ' '}" f" {status_color}{str(e['answers']):<40}{RESET}" f"{info}" ) print(line) - if e["status"] == "fail" or e["status"] == "error": + if e["status"] == "missing" or e["status"] == "error": problems.append(line) - if not e["cache"] and e["elapsed"] / 1e9 > 5: + if not in_cache and e["elapsed"] / 1e9 > 5: save_cache() results.add(" ".join(e["answers"])) @@ -298,11 +328,13 @@ def main(): parser.add_argument("-l", "--language", type=str, metavar="LANG", help="filter by language") parser.add_argument("-r", "--refresh", action="store_true", help="relaunch solutions") parser.add_argument("-n", "--dry-run", action="store_true", help="do not run") + parser.add_argument("--no-build", action="store_true", help="do not build") + parser.add_argument("-u", "--user", dest="filter_user", type=str, help="filter by user id") parser.add_argument("n", type=int, nargs="*", help="filter by year or year/day") args = parser.parse_args() - filter_year = 0 if len(args.n) == 0 else args.n.pop(0) + filter_year = 0 if len(args.n) == 0 else int(args.n.pop(0)) filter_day = set(args.n) if args.language == "cpp": args.language = "c++" @@ -313,8 +345,11 @@ def main(): problems = [] stats_elapsed = dict() - build_all(filter_year) - inputs, sols = load_data(filter_year) + if not args.no_build: + build_all(filter_year) + print(end=f"{CR}{CLEAR_EOL}") + + inputs, sols = load_data(filter_year, args.filter_user) for year in range(2015, 2024): if filter_year != 0 and year != filter_year: