Skip to content

Commit

Permalink
scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
rene-d committed Feb 2, 2024
1 parent 093dabf commit 7878f98
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 84 deletions.
8 changes: 8 additions & 0 deletions scripts/Dockerfile.pypy3
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM pypy:3-bookworm

RUN apt-get update && \
apt-get upgrade -y && \
apt-get install -y libgeos-dev

COPY requirements.txt /
RUN pip install -r requirements.txt
203 changes: 119 additions & 84 deletions scripts/runall.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from operator import itemgetter
from pathlib import Path
from zlib import crc32
import sqlite3

RED = "\033[91m"
GREEN = "\033[92m"
Expand All @@ -38,108 +39,122 @@
"C++": "{year}/build/day{day}_cpp",
}

INTERPRETERS = {
"Python": "python3",
"PyPy": "pypy3",
}


def get_cache():
"""Retrieve the cache instance from memory or load it from disk."""
cache = globals().get("_cache")
if cache is None:
cache_file = Path(__file__).parent.parent / "data" / "cache.json"
if cache_file.is_file():
cache = json.loads(cache_file.read_bytes())
else:
cache = {}
cache_file = Path(__file__).parent.parent / "data" / "cache.db"

cache_db = sqlite3.connect(cache_file)

cache_db.executescript(
"create table if not exists solutions ("
" key text primary key not null,"
" mtime_ns int,"
" elapsed float,"
" status text,"
" answers text);"
)
cache = {"db": cache_db, "modified": False}
globals()["_cache"] = cache
cache["modified"] = False

return cache


def save_cache():
cache = get_cache()
if cache["modified"]:
cache.pop("modified")
cache_file = Path(__file__).parent.parent / "data" / "cache.json"
cache_file.write_text(json.dumps(cache, indent=2, ensure_ascii=True))
cache["modified"] = False
print(f"{FEINT}{ITALIC}cache commited{RESET}")
pass
# cache = get_cache()
# if cache["modified"]:
# cache["db"].commit()
# cache["modified"] = False
# print(f"{FEINT}{ITALIC}cache commited{RESET}")


def check_cache(key, file_timestamp: Path, no_check=False):
def check_cache(key, file_timestamp: Path, no_age_check=False):
cache = get_cache()
key = str(key)
e = cache.get(key, None)
if e:
db = cache["db"]
cursor = db.execute("select mtime_ns,elapsed,status,answers from solutions where key=?", (key,))
row = cursor.fetchone()
if row:
timestamp = file_timestamp.stat().st_mtime_ns
if e["timestamp"] == timestamp or no_check:
return e
if row[0] == timestamp or no_age_check:
return {
"elapsed": row[1],
"status": row[2],
"answers": row[3].split("\n"),
}
else:
seconds = round((timestamp - e["timestamp"]) / 1000000000)
delta = timedelta(seconds=seconds)
# seconds = round((timestamp - e["timestamp"]) / 1000000000)
# delta = timedelta(seconds=seconds)
# print(f"{FEINT}{ITALIC}entry {key} is out of date for {delta}{RESET}", end=f"{CR}")

print(f"{FEINT}{ITALIC}entry {key} is out of date for {delta}{RESET}", end=f"{CR}")
print(f"{FEINT}{ITALIC}entry {key} is out of date{RESET}", end=f"{CR}")

else:
print(f"{FEINT}{ITALIC}missing cache for {key}{RESET}", end=f"{CR}")


def update_cache(key, timestamp: Path, elapsed, status, answers):
def update_cache(key, timestamp: Path, elapsed: float, status: str, answers: t.Iterable):
cache = get_cache()
db = cache["db"]
key = str(key)
e = cache.get(key, {})
e["timestamp"] = timestamp.stat().st_mtime_ns
e["elapsed"] = elapsed
e["status"] = status
e["answers"] = answers
cache[key] = e
cache["modified"] = True
return e

db.execute(
"insert or replace into solutions (key,mtime_ns,elapsed,status,answers) values (?,?,?,?,?)",
(key, timestamp.stat().st_mtime_ns, elapsed, status, "\n".join(answers)),
)

# cache["modified"] = True
db.commit()

def run(key: str, prog: Path, lang: str, file: Path, solution: t.List, refresh: bool, dry_run: bool):
return {
"elapsed": elapsed,
"status": status,
"answers": answers,
}


def run(prog: Path, lang: str, file: Path, solution: t.List, warmup: bool) -> t.Dict[str, t.Any]:
if not prog.is_file():
return

cmd = [prog.absolute().as_posix()]

if lang == "Python":
cmd.insert(0, "python3")
elif lang == "PyPy":
cmd.insert(0, "pypy3")

if refresh:
e = None
else:
e = check_cache(key, prog, dry_run)
if dry_run and not e:
return None
if e:
in_cache = True
# add the interpreter
interpreter = INTERPRETERS.get(lang)
if interpreter:
cmd.insert(0, interpreter)

if warmup and lang == "Rust":
# under macOS, the first launch of a Rust program is slower (why ???)
subprocess.call(cmd + ["--help"], stdout=subprocess.DEVNULL)

start = time.time_ns()
out = subprocess.run(cmd + [file.absolute()], stdout=subprocess.PIPE)
elapsed = time.time_ns() - start
answers = out.stdout.decode().strip()

status = "unknown"
if solution:
solution = solution.read_text()
if answers == solution.strip():
status = "ok"
else:
status = "error"
else:
in_cache = False

start = time.time_ns()
out = subprocess.run(cmd + [file.absolute()], stdout=subprocess.PIPE)
elapsed = time.time_ns() - start
answers = out.stdout.decode().strip()

status = "unknown"
if solution:
solution = solution.read_text()
if answers == solution.strip():
status = "ok"
else:
status = "error"
if answers == "":
status = "missing"
else:
if answers == "":
status = "fail"
else:
status = "unknown"

e = update_cache(key, prog, elapsed, status, answers.split("\n"))

e = deepcopy(e)
e["cache"] = in_cache
status = "unknown"

return e
return {"elapsed": elapsed, "status": status, "answers": answers.split("\n")}


def make(year: Path, source: Path, dest: Path, cmd: str):
Expand All @@ -159,7 +174,7 @@ def make(year: Path, source: Path, dest: Path, cmd: str):
subprocess.check_call(cmdline, shell=True)


def build_all(filter_year):
def build_all(filter_year: int):
for year in range(2015, 2024):
if filter_year != 0 and year != filter_year:
continue
Expand All @@ -183,13 +198,20 @@ def build_all(filter_year):
make(year, src, f"day{day}_cpp", "c++ -std=c++17")


def load_data(filter_year):
def load_data(filter_year, filter_user):
inputs = defaultdict(dict)
solutions = defaultdict(dict)

for f in Path("data").rglob("*.in"):

if f.name.startswith("._"):
continue

assert len(f.parts) == 4

if filter_user and f.parent.parent.name != filter_user:
continue

year = int(f.parent.name)
day = int(f.stem)

Expand All @@ -201,7 +223,7 @@ def load_data(filter_year):
crc = e["status"]
else:
crc = hex(crc32(f.read_bytes().strip()) & 0xFFFFFFFF)
update_cache(f, f, 0, crc, 0)
update_cache(f, f, 0, crc, [])

if crc not in inputs[year, day]:
inputs[year, day][crc] = f
Expand All @@ -219,7 +241,7 @@ def run_day(
):
elapsed = defaultdict(list)

first = True
warmup = defaultdict(lambda: True)

day_suffix = mday.removeprefix(str(day))
name_max_len = 16 - len(day_suffix)
Expand All @@ -242,12 +264,20 @@ def run_day(
prog = Path(pattern.format(year=year, day=mday))
key = ":".join(map(str, (year, day, crc, prog, lang.lower())))

if lang.lower() == "rust" and first and prog.is_file():
# under macOS, the first launch of a program is slower
first = False
subprocess.call([prog, "--help"], stdout=subprocess.DEVNULL)
if refresh:
e = None
in_cache = False
else:
e = check_cache(key, prog, dry_run)
in_cache = e is not None

if not in_cache and not dry_run:

e = run(prog, lang, file, day_sols.get(crc), warmup[lang])

e = run(key, prog, lang, file, day_sols.get(crc), refresh, dry_run)
if e:
warmup[lang] = False
e = update_cache(key, prog, e["elapsed"], e["status"], e["answers"])

if not e:
continue
Expand All @@ -257,24 +287,24 @@ def run_day(
else:
info = ""

status_color = {"fail": MAGENTA, "unknown": GRAY, "error": RED, "ok": GREEN}[e["status"]]
status_color = {"missing": MAGENTA, "unknown": GRAY, "error": RED, "ok": GREEN}[e["status"]]

line = (
f"{CR}{RESET}{CLEAR_EOL}"
f"{prefix}"
f" {YELLOW}{lang:<7}{RESET}:"
f" {status_color}{e['status']:7}{RESET}"
f" {WHITE}{e['elapsed']/1e9:7.3f}s"
f" {GRAY}{'☽' if e['cache'] else ' '}"
f" {GRAY}{'☽' if in_cache else ' '}"
f" {status_color}{str(e['answers']):<40}{RESET}"
f"{info}"
)
print(line)

if e["status"] == "fail" or e["status"] == "error":
if e["status"] == "missing" or e["status"] == "error":
problems.append(line)

if not e["cache"] and e["elapsed"] / 1e9 > 5:
if not in_cache and e["elapsed"] / 1e9 > 5:
save_cache()

results.add(" ".join(e["answers"]))
Expand All @@ -298,11 +328,13 @@ def main():
parser.add_argument("-l", "--language", type=str, metavar="LANG", help="filter by language")
parser.add_argument("-r", "--refresh", action="store_true", help="relaunch solutions")
parser.add_argument("-n", "--dry-run", action="store_true", help="do not run")
parser.add_argument("--no-build", action="store_true", help="do not build")
parser.add_argument("-u", "--user", dest="filter_user", type=str, help="filter by user id")
parser.add_argument("n", type=int, nargs="*", help="filter by year or year/day")

args = parser.parse_args()

filter_year = 0 if len(args.n) == 0 else args.n.pop(0)
filter_year = 0 if len(args.n) == 0 else int(args.n.pop(0))
filter_day = set(args.n)
if args.language == "cpp":
args.language = "c++"
Expand All @@ -313,8 +345,11 @@ def main():
problems = []
stats_elapsed = dict()

build_all(filter_year)
inputs, sols = load_data(filter_year)
if not args.no_build:
build_all(filter_year)
print(end=f"{CR}{CLEAR_EOL}")

inputs, sols = load_data(filter_year, args.filter_user)

for year in range(2015, 2024):
if filter_year != 0 and year != filter_year:
Expand Down

0 comments on commit 7878f98

Please sign in to comment.