Skip to content

Commit

Permalink
improved language support
Browse files Browse the repository at this point in the history
  • Loading branch information
@rene-d
rene-d committed Feb 3, 2024
1 parent 5c21353 commit c63bbe1
Showing 1 changed file with 155 additions and 58 deletions.
213 changes: 155 additions & 58 deletions scripts/runall.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import argparse
import itertools
import json
import os
import subprocess
import time
Expand All @@ -14,6 +13,7 @@
from pathlib import Path
from zlib import crc32
import sqlite3
import shutil

RED = "\033[91m"
GREEN = "\033[92m"
Expand All @@ -33,17 +33,20 @@

LANGUAGES = {
"Python": "{year}/day{day}/day{day}.py",
"PyPy": "{year}/day{day}/day{day}.py",
"Rust": "{year}/target/release/day{day}",
"C": "{year}/build/day{day}_c",
"C++": "{year}/build/day{day}_cpp",
}

DEFAULT_LANGUAGES = {"Python", "Rust"}

INTERPRETERS = {
"Python": "python3",
"PyPy": "pypy3",
"Python": {
"Python": "python3",
"PyPy": ".pypy3.10/bin/python",
"Py3.10": ".py3.10/bin/python",
"Py3.11": ".py3.11/bin/python",
"Py3.12": ".py3.12/bin/python",
"Py3.13": ".py3.13/bin/python",
}
}


Expand All @@ -61,7 +64,13 @@ def get_cache():
" mtime_ns int,"
" elapsed float,"
" status text,"
" answers text);"
" answers text"
");"
"create table if not exists inputs ("
" key text primary key not null,"
" mtime_ns int,"
" crc32 text"
");"
)
cache = {"db": cache_db, "modified": False}
globals()["_cache"] = cache
Expand All @@ -78,20 +87,18 @@ def save_cache():
# print(f"{FEINT}{ITALIC}cache commited{RESET}")


def check_cache(key, file_timestamp: Path, no_age_check=False):
def check_cache(key, file_timestamp: Path, table: str, columns: t.Iterable[str], no_age_check=False):
cache = get_cache()
key = str(key)
db = cache["db"]
cursor = db.execute("select mtime_ns,elapsed,status,answers from solutions where key=?", (key,))
db.row_factory = sqlite3.Row
cursor = db.execute(f"select * from `{table}` where key=?", (key,))
row = cursor.fetchone()
if row:
timestamp = file_timestamp.stat().st_mtime_ns
if row[0] == timestamp or no_age_check:
return {
"elapsed": row[1],
"status": row[2],
"answers": row[3].split("\n"),
}
if row["mtime_ns"] == timestamp or no_age_check:
return dict((column, row[column]) for column in columns)

else:
# seconds = round((timestamp - e["timestamp"]) / 1000000000)
# delta = timedelta(seconds=seconds)
Expand All @@ -103,34 +110,40 @@ def check_cache(key, file_timestamp: Path, no_age_check=False):
print(f"{FEINT}{ITALIC}missing cache for {key}{RESET}", end=f"{CLEAR_EOL}{CR}")


def update_cache(key, timestamp: Path, elapsed: float, status: str, answers: t.Iterable):
def update_cache(key, timestamp: Path, table: str, row: t.Dict[str, t.Union[str, int]]) -> None:
cache = get_cache()
db = cache["db"]
key = str(key)

db.execute(
"insert or replace into solutions (key,mtime_ns,elapsed,status,answers) values (?,?,?,?,?)",
(key, timestamp.stat().st_mtime_ns, elapsed, status, "\n".join(answers)),
)
sql = f"insert or replace into `{table}` (key,mtime_ns"
values = [str(key), timestamp.stat().st_mtime_ns]

# cache["modified"] = True
db.commit()
for k, v in row.items():
sql += f",`{k}`"
values.append(v)

return {
"elapsed": elapsed,
"status": status,
"answers": answers,
}
sql += ") values (?,?"
sql += ",?" * len(row)
sql += ")"

db.execute(sql, values)

db.commit()


def run(prog: Path, lang: str, file: Path, solution: t.List, warmup: bool) -> t.Dict[str, t.Any]:
def run(
prog: Path,
lang: str,
interpreter: t.Union[None, str],
file: Path,
solution: t.List,
warmup: bool,
) -> t.Dict[str, t.Any]:
if not prog.is_file():
return

cmd = [prog.absolute().as_posix()]

# add the interpreter
interpreter = INTERPRETERS.get(lang)
if interpreter:
cmd.insert(0, interpreter)

Expand All @@ -156,7 +169,7 @@ def run(prog: Path, lang: str, file: Path, solution: t.List, warmup: bool) -> t.
else:
status = "unknown"

return {"elapsed": elapsed, "status": status, "answers": answers.split("\n")}
return {"elapsed": elapsed, "status": status, "answers": answers}


def make(year: Path, source: Path, dest: Path, cmd: str):
Expand Down Expand Up @@ -211,7 +224,9 @@ def load_data(filter_year, filter_user):

assert len(f.parts) == 4

if filter_user and f.parent.parent.name != filter_user:
user = f.parent.parent.name

if filter_user and user != filter_user:
continue

year = int(f.parent.name)
Expand All @@ -220,12 +235,14 @@ def load_data(filter_year, filter_user):
if filter_year != 0 and year != filter_year:
continue

e = check_cache(f, f)
key = f"{year}:{day}:{user}"

e = check_cache(key, f, "inputs", ("crc32",))
if e:
crc = e["status"]
crc = e["crc32"]
else:
crc = hex(crc32(f.read_bytes().strip()) & 0xFFFFFFFF)
update_cache(f, f, 0, crc, [])
update_cache(key, f, "inputs", {"crc32": crc})

if crc not in inputs[year, day]:
inputs[year, day][crc] = f
Expand All @@ -239,7 +256,15 @@ def load_data(filter_year, filter_user):


def run_day(
year: int, day: int, mday: str, day_inputs: t.Dict, day_sols: t.Dict, problems: t.Set, filter_lang, refresh, dry_run
year: int,
day: int,
mday: str,
day_inputs: t.Dict,
day_sols: t.Dict,
languages: dict,
problems: t.Set,
refresh: bool,
dry_run: bool,
):
elapsed = defaultdict(list)

Expand All @@ -259,14 +284,7 @@ def run_day(

results = set()

for lang, pattern in LANGUAGES.items():
if filter_lang == "all":
pass
elif filter_lang:
if lang.lower() != filter_lang.lower():
continue
elif lang not in DEFAULT_LANGUAGES:
continue
for lang, (pattern, interpreter) in languages.items():

prog = Path(pattern.format(year=year, day=mday))
key = ":".join(map(str, (year, day, crc, prog, lang.lower())))
Expand All @@ -275,16 +293,16 @@ def run_day(
e = None
in_cache = False
else:
e = check_cache(key, prog, dry_run)
e = check_cache(key, prog, "solutions", ("elapsed", "status", "answers"), dry_run)
in_cache = e is not None

if not in_cache and not dry_run:

e = run(prog, lang, file, day_sols.get(crc), warmup[lang])
e = run(prog, lang, interpreter, file, day_sols.get(crc), warmup[lang])

if e:
warmup[lang] = False
e = update_cache(key, prog, e["elapsed"], e["status"], e["answers"])
update_cache(key, prog, "solutions", e)

if not e:
continue
Expand All @@ -296,14 +314,16 @@ def run_day(

status_color = {"missing": MAGENTA, "unknown": GRAY, "error": RED, "ok": GREEN}[e["status"]]

answers = e["answers"].split("\n")

line = (
f"{CR}{RESET}{CLEAR_EOL}"
f"{prefix}"
f" {YELLOW}{lang:<7}{RESET}:"
f" {status_color}{e['status']:7}{RESET}"
f" {WHITE}{e['elapsed']/1e9:7.3f}s"
f" {GRAY}{'☽' if in_cache else ' '}"
f" {status_color}{str(e['answers']):<40}{RESET}"
f" {status_color}{str(answers):<40}{RESET}"
f"{info}"
)
print(line)
Expand All @@ -314,7 +334,7 @@ def run_day(
if not in_cache and e["elapsed"] / 1e9 > 5:
save_cache()

results.add(" ".join(e["answers"]))
results.add(" ".join(answers))

elapsed[lang].append(e["elapsed"] / 1e9)

Expand All @@ -330,9 +350,83 @@ def run_day(
return dict((lang, sum(t) / len(t)) for lang, t in elapsed.items()), nb_samples


def get_languages(filter_lang: t.Iterable[str]) -> t.Dict[str, t.Tuple[str, t.Union[str, None]]]:

filter_lang = set(map(str.casefold, filter_lang or ()))

languages = {}
for lang, v in LANGUAGES.items():

if lang in INTERPRETERS:
for lang2, interpreter in INTERPRETERS[lang].items():

if filter_lang and lang2.casefold() not in filter_lang:
continue

if "/" not in interpreter and "\\" not in interpreter:
interpreter = shutil.which(interpreter)
languages[lang2] = (v, interpreter)
else:
interpreter = Path(interpreter).expanduser().absolute()
if interpreter.is_file() and (interpreter.stat().st_mode & os.X_OK) != 0:
languages[lang2] = (v, interpreter.as_posix())
else:
# print(f"language {lang2} : interpreter {interpreter} not found")
pass
else:

if filter_lang and lang.casefold() not in filter_lang:
continue
languages[lang] = (v, None)

return languages


def install_venv(interpreter: Path):

try:
slug = 'import sys;print(((hasattr(sys, "subversion") and getattr(sys, "subversion")) or ("Py",))[0] + f"{sys.version_info.major}.{sys.version_info.minor}")'

slug = subprocess.check_output([interpreter, "-c", slug]).decode().strip()

venv = "." + slug.lower()

# subprocess.check_call([interpreter, "-mensurepip"])
subprocess.check_output([interpreter, "-mvenv", venv])
subprocess.check_call(
[
f"{venv}/bin/python3",
"-mpip",
"install",
"--no-input",
"--quiet",
"--upgrade",
"pip",
]
)
subprocess.check_call(
[
f"{venv}/bin/python3",
"-mpip",
"install",
"--no-input",
"--quiet",
"--upgrade",
"-r",
"scripts/requirements.txt",
]
)

print(f"Virtual environment for {MAGENTA}{interpreter}{RESET} installed into: {GREEN}{venv}{RESET}")

except subprocess.CalledProcessError as e:
print(e)


def main():
parser = argparse.ArgumentParser()
parser.add_argument("-l", "--language", type=str, metavar="LANG", help="filter by language")
parser.add_argument("--venv", type=Path, help="install virtual environment")
parser.add_argument("-l", "--language", type=str, action="append", metavar="LANG", help="filter by language")
parser.add_argument("-r", "--refresh", action="store_true", help="relaunch solutions")
parser.add_argument("-n", "--dry-run", action="store_true", help="do not run")
parser.add_argument("--no-build", action="store_true", help="do not build")
Expand All @@ -341,21 +435,24 @@ def main():

args = parser.parse_args()

filter_year = 0 if len(args.n) == 0 else int(args.n.pop(0))
filter_day = set(args.n)
if args.language == "cpp":
args.language = "c++"

try:
os.chdir(Path(__file__).parent.parent)

problems = []
stats_elapsed = dict()

os.chdir(Path(__file__).parent.parent)

if args.venv:
return install_venv(args.venv)

filter_year = 0 if len(args.n) == 0 else int(args.n.pop(0))
filter_day = set(args.n)

if not args.no_build:
build_all(filter_year)
print(end=f"{CR}{CLEAR_EOL}")

languages = get_languages(args.language)

inputs, sols = load_data(filter_year, args.filter_user)

for year in range(2015, 2024):
Expand All @@ -375,8 +472,8 @@ def main():
mday,
inputs[year, day],
sols[year, day],
languages,
problems,
args.language,
args.refresh,
args.dry_run,
)
Expand Down

0 comments on commit c63bbe1

Please sign in to comment.