diff --git a/.github/workflows/cli-lint.yml b/.github/workflows/cli-lint.yml index 28413d167..36defd270 100644 --- a/.github/workflows/cli-lint.yml +++ b/.github/workflows/cli-lint.yml @@ -49,15 +49,8 @@ jobs: steps: - uses: actions/checkout@v4 - # the codegen uses `rustfmt +nightly` - - uses: dtolnay/rust-toolchain@nightly - with: - components: rustfmt - - name: Regen openapi libs - run: | - yarn - ./regen_openapi.sh + run: ./regen_openapi.py - uses: dtolnay/rust-toolchain@master with: diff --git a/.github/workflows/codegen.yml b/.github/workflows/codegen.yml index 84b68a5b8..7f91d00ab 100644 --- a/.github/workflows/codegen.yml +++ b/.github/workflows/codegen.yml @@ -25,34 +25,8 @@ jobs: steps: - uses: actions/checkout@v4 - # JS codegen uses biome - - name: Setup Biome - uses: biomejs/setup-biome@v2 - with: - version: "1.9.4" - - # Rust codegen uses `rustfmt +nightly` - - uses: dtolnay/rust-toolchain@nightly - with: - components: rustfmt - - # Python codegen uses `ruff` for formatting and cleaning up unused imports - - run: pipx install ruff - - # cache the openapi-codegen binary installed by regen_openapi.sh - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "rust -> target" - # only save the cache on the main branch - # cf https://github.com/Swatinem/rust-cache/issues/95 - save-if: ${{ github.ref == 'refs/heads/main' }} - # include relevant information in the cache name - prefix-key: "codegen-${{ matrix.rust }}" - - name: Regen openapi libs - run: | - yarn - ./regen_openapi.sh + run: ./regen_openapi.py - name: Check for uncommitted changes run: | diff --git a/.github/workflows/rust-lint.yml b/.github/workflows/rust-lint.yml index f608da869..185bff7b4 100644 --- a/.github/workflows/rust-lint.yml +++ b/.github/workflows/rust-lint.yml @@ -47,15 +47,8 @@ jobs: steps: - uses: actions/checkout@v4 - # the codegen uses `rustfmt +nightly` - - uses: dtolnay/rust-toolchain@nightly - with: - components: rustfmt - - name: Regen openapi libs - run: | - yarn - ./regen_openapi.sh + run: ./regen_openapi.py - uses: dtolnay/rust-toolchain@master with: diff --git a/codegen.toml b/codegen.toml new file mode 100644 index 000000000..f707cf86e --- /dev/null +++ b/codegen.toml @@ -0,0 +1,129 @@ +[global] +openapi = "lib-openapi.json" + +[rust] +template_dir = "rust/templates" +extra_mounts = { "rust/.rustfmt.toml" = "/app/.rustfmt.toml" } +extra_shell_commands = ["rm rust/src/api/{environment,health}.rs"] +[[rust.task]] +template = "rust/templates/api_resource.rs.jinja" +output_dir = "rust/src/api" +[[rust.task]] +template = "rust/templates/component_type.rs.jinja" +output_dir = "rust/src/models" + + +[javascript] +template_dir = "javascript/templates" +extra_shell_commands = ["rm javascript/src/api/{ingest,operationalWebhook}.ts"] +[[javascript.task]] +template = "javascript/templates/api_resource.ts.jinja" +output_dir = "javascript/src/api" +[[javascript.task]] +template = "javascript/templates/component_type_summary.ts.jinja" +output_dir = "javascript/src/models" +[[javascript.task]] +template = "javascript/templates/component_type.ts.jinja" +output_dir = "javascript/src/models" + + +[cli] +template_dir = "svix-cli/templates" +extra_mounts = { "svix-cli/.rustfmt.toml" = "/app/.rustfmt.toml" } +extra_shell_commands = [ + "cargo fix --manifest-path svix-cli/Cargo.toml --allow-dirty", + "cargo fmt --manifest-path svix-cli/Cargo.toml", + "rm svix-cli/src/cmds/api/{ingest,operational_webhook,background_task,environment,health,operational_webhook_endpoint,statistics}.rs", +] +[[cli.task]] +template = "svix-cli/templates/api_resource.rs.jinja" +output_dir = "svix-cli/src/cmds/api" + + +[python] +template_dir = "python/templates" +extra_shell_commands = [ + "rm python/svix/api/{environment,health,ingest,operational_webhook}.py", +] +[[python.task]] +template = "python/templates/api_resource.py.jinja" +output_dir = "python/svix/api" +[[python.task]] +template = "python/templates/component_type_summary.py.jinja" +output_dir = "python/svix/models" +[[python.task]] +template = "python/templates/component_type.py.jinja" +output_dir = "python/svix/models" + + +[ruby] +template_dir = "ruby/templates" +extra_shell_commands = ["rm ruby/lib/svix/api/{ingest,operational_webhook}.rb"] +[[ruby.task]] +template = "ruby/templates/api_resource.rb.jinja" +output_dir = "ruby/lib/svix/api" +[[ruby.task]] +template = "ruby/templates/summary.rb.jinja" +output_dir = "ruby/lib" +[[ruby.task]] +template = "ruby/templates/component_type.rb.jinja" +output_dir = "ruby/lib/svix/models" + + +[csharp] +template_dir = "csharp/templates" +extra_shell_commands = [ + "rm csharp/Svix/{IngestEndpoint,Ingest,OperationalWebhook}.cs", +] +[[csharp.task]] +template = "csharp/templates/api_resource.cs.jinja" +output_dir = "csharp/Svix" +[[csharp.task]] +template = "csharp/templates/component_type.cs.jinja" +output_dir = "csharp/Svix/Models" + + +[java] +template_dir = "java/templates" +extra_shell_commands = [ + "rm java/lib/src/main/java/com/svix/api/{OperationalWebhook,Ingest}.java", +] +[[java.task]] +template = "java/templates/api_resource.java.jinja" +output_dir = "java/lib/src/main/java/com/svix/api" +[[java.task]] +template = "java/templates/operation_options.java.jinja" +output_dir = "java/lib/src/main/java/com/svix/api" +[[java.task]] +template = "java/templates/component_type.java.jinja" +output_dir = "java/lib/src/main/java/com/svix/models" + + +[go] +template_dir = "openapi-templates/go" +extra_shell_commands = [ + "rm go/{environment,health,ingest_endpoint,ingest,operational_webhook}.go", +] +[[go.task]] +template = "openapi-templates/go/api_resource.go.jinja" +output_dir = "go" +[[go.task]] +template = "openapi-templates/go/component_type_summary.go.jinja" +output_dir = "go" +[[go.task]] +template = "openapi-templates/go/component_type.go.jinja" +output_dir = "go/models" + + +[kotlin] +extra_shell_commands = [ + "rm kotlin/lib/src/main/kotlin/{Ingest,OperationalWebhook}.kt", +] +template_dir = "kotlin/templates" +[[kotlin.task]] +template = "kotlin/templates/component_type.kt.jinja" +output_dir = "kotlin/lib/src/main/kotlin/models" + +[[kotlin.task]] +template = "kotlin/templates/api_resource.kt.jinja" +output_dir = "kotlin/lib/src/main/kotlin" diff --git a/package.json b/package.json index 80e25c0b7..aecea76c2 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "scripts": { "bump-version": "node tools/bump_version.js", "update-postman": "node tools/update_postman.js", - "generate": "bash ./regen_openapi.sh" + "generate": "python3 ./regen_openapi.py" }, "devDependencies": { "axios": "^0.28.0", diff --git a/regen_openapi.py b/regen_openapi.py new file mode 100755 index 000000000..f209a8434 --- /dev/null +++ b/regen_openapi.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +import json +import os +import pathlib +import random +import shutil +import string +import subprocess +from pathlib import Path +from threading import Thread + +try: + import tomllib +except ImportError: + print("Python 3.11 or greater is required to run the codegen") + exit(1) + +OPENAPI_CODEGEN_IMAGE = "ghcr.io/svix/openapi-codegen:20250303-a07aa78" +REPO_ROOT = pathlib.Path(__file__).parent.resolve() +DEBUG = os.getenv("DEBUG") is not None +GREEN = "\033[92m" +BLUE = "\033[94m" +CYAN = "\033[96m" +ENDC = "\033[0m" + + +def get_docker_binary() -> str: + # default to podman + docker_binary = shutil.which("podman") + if docker_binary is None: + docker_binary = shutil.which("docker") + if docker_binary is None: + print("Please install docker or podman to run the codegen") + exit(1) + return docker_binary + + +def docker_container_rm(prefix, container_id): + cmd = [get_docker_binary(), "container", "rm", container_id] + result = run_cmd(prefix, cmd) + return result.stdout.decode("utf-8") + + +def docker_container_logs(prefix, container_id): + cmd = [get_docker_binary(), "container", "logs", container_id] + result = run_cmd(prefix, cmd, dont_dbg=True) + return f"{result.stdout.decode('utf-8')}\n{result.stderr.decode('utf-8')}".strip() + + +def docker_container_wait(prefix, container_id) -> int: + cmd = [get_docker_binary(), "container", "wait", container_id] + result = run_cmd(prefix, cmd) + return int(result.stdout.decode("utf-8")) + + +def docker_container_cp(prefix, container_id, task): + cmd = [ + get_docker_binary(), + "container", + "cp", + f"{container_id}:/app/{task['output_dir']}/.", + f"{task['output_dir']}/", + ] + run_cmd(prefix, cmd) + + +def docker_container_create(prefix, task) -> str: + container_name = "codegen-{}-{}-{}".format( + task["language"], + task["language_task_index"] + 1, + "".join(random.choice(string.ascii_lowercase) for _ in range(10)), + ) + cmd = [ + get_docker_binary(), + "container", + "run", + "-d", + "--name", + container_name, + "--workdir", + "/app", + "--mount", + f"type=bind,src={Path(task['openapi']).absolute()},dst=/app/lib-openapi.json,ro", + "--mount", + f"type=bind,src={Path(task['template_dir']).absolute()},dst=/app/{task['template_dir']},ro", + ] + + for extra_mount_src, extra_mount_dst in task["extra_mounts"].items(): + cmd.append("--mount") + cmd.append( + f"type=bind,src={Path(extra_mount_src).absolute()},dst={extra_mount_dst},ro" + ) + cmd.extend( + [ + OPENAPI_CODEGEN_IMAGE, + "openapi-codegen", + "generate", + *task["extra_codegen_args"], + "--template", + task["template"], + "--input-file", + task["openapi"], + "--output-dir", + task["output_dir"], + ] + ) + run_cmd(prefix, cmd) + return container_name + + +def run_cmd(prefix, cmd, dont_dbg=False) -> subprocess.CompletedProcess[bytes]: + dbg_cmd = [cmd[0], *[f'"{i}"' for i in cmd[1:]]] + dbg(prefix, f"{BLUE}Running command{ENDC} {' '.join(dbg_cmd)}") + result = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=REPO_ROOT + ) + if result.returncode != 0: + print_cmd_result(result, prefix) + exit(result.returncode) + + if DEBUG and not dont_dbg: + print_cmd_result(result, prefix) + + return result + + +def print_cmd_result(result: subprocess.CompletedProcess[bytes], prefix: str): + for i, stream in enumerate([result.stdout, result.stderr]): + output = stream.decode("utf-8") + if output != "": + cli_prefix = f"{CYAN}{'stdout' if i == 0 else 'stderr'}{ENDC} " + nice_logs = "{}{}".format( + cli_prefix, + output.strip().replace("\n", f"\n{cli_prefix}"), + ) + prefix_print(prefix, nice_logs) + + +def dbg(prefix, msg): + if not DEBUG: + return + prefix_print(prefix, msg) + + +def prefix_print(prefix, msg): + print( + "{}{}".format(f"{prefix.strip()} ", msg.replace("\n", f"\n{prefix.strip()} ")), + flush=True, + ) + + +def execute_codegen_task(task): + prefix = "{}{} ({}/{}){} | ".format( + GREEN, + task["language"], + task["language_task_index"] + 1, + task["language_total"], + ENDC, + ) + + prefix_print(prefix, "Starting codegen task") + + container_name = docker_container_create(prefix, task).strip() + dbg(prefix, f"Container id {container_name}") + + exit_code = docker_container_wait(prefix, container_name) + + logs = docker_container_logs(prefix, container_name) + + nice_logs = "{}{}".format( + f"{CYAN}container logs{ENDC} ", + logs.strip().replace("\n", f"\n{CYAN}container logs{ENDC} "), + ) + + if exit_code != 0: + prefix_print(prefix, nice_logs) + raise RuntimeError(f"Container exited with {exit_code}") + + dbg(prefix, nice_logs) + + docker_container_cp(prefix, container_name, task) + + docker_container_rm(prefix, container_name) + + prefix_print(prefix, "Codegen task completed") + + +def run_codegen_for_language(language, language_config): + threads = [] + for t in language_config["tasks"]: + th = Thread(target=execute_codegen_task, args=[t]) + th.start() + threads.append(th) + + for th in threads: + th.join() + + extra_shell_commands = language_config.get("extra_shell_commands", []) + for index, shell_command in enumerate(extra_shell_commands): + cmd = ["bash", "-c", shell_command] + run_cmd( + f"{GREEN}{language}[extra shell commands] ({index + 1}/{len(extra_shell_commands)}){ENDC} | ", + cmd, + ) + + +def parse_config(): + with open(REPO_ROOT.joinpath("codegen.toml"), "rb") as f: + data = tomllib.load(f) + openapi = data.pop("global")["openapi"] + config = {} + for language, language_config in data.items(): + config[language] = {"tasks": []} + for language_task_index, task in enumerate(language_config["task"]): + config[language]["extra_shell_commands"] = language_config.get( + "extra_shell_commands", [] + ) + config[language]["tasks"].append( + { + "language": language, + "language_task_index": language_task_index, + "language_total": len(language_config.get("task", [])), + "openapi": task.get( + "openapi", language_config.get("openapi", openapi) + ), + "template": task["template"], + "output_dir": task["output_dir"], + "extra_mounts": language_config.get("extra_mounts", {}), + "extra_codegen_args": task.get("extra_codegen_args", []), + "template_dir": language_config["template_dir"], + } + ) + if DEBUG: + print(json.dumps(config, indent=4), flush=True) + return config + + +def main(): + config = parse_config() + print("Pulling docker image", flush=True) + run_cmd("startup", [get_docker_binary(), "image", "pull", OPENAPI_CODEGEN_IMAGE]) + + threads = [] + for language, language_config in config.items(): + th = Thread(target=run_codegen_for_language, args=[language, language_config]) + th.start() + threads.append(th) + + for th in threads: + th.join() + + +if __name__ == "__main__": + main() diff --git a/regen_openapi.sh b/regen_openapi.sh deleted file mode 100755 index d5c63f1be..000000000 --- a/regen_openapi.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -OPENAPI_GIT_REV='272125558d6ac4718bdc87b1652e5d4122b69f19' - -if [ -n "$1" ]; then - curl "$1" | python -m json.tool > lib-openapi.json -fi - -if ! command -v openapi-codegen >/dev/null; then - if [[ -z "$GITHUB_WORKFLOW" ]]; then - echo openapi-codegen is not installed. install using - echo "cargo install --git https://github.com/svix/openapi-codegen --rev $OPENAPI_GIT_REV --locked" - exit 1 - else - cargo install --git https://github.com/svix/openapi-codegen --rev $OPENAPI_GIT_REV --locked - fi -fi - -# Print commands we run from here on -set -x - -# === Kotlin ==== -if [[ -z "$CI" ]]; then - openapi-codegen generate \ - --template kotlin/templates/api_resource.kt.jinja \ - --input-file lib-openapi.json \ - --output-dir kotlin/lib/src/main/kotlin - openapi-codegen generate \ - --template kotlin/templates/component_type.kt.jinja \ - --input-file lib-openapi.json \ - --output-dir kotlin/lib/src/main/kotlin/models -fi - - -# === Go === -if [[ -z "$CI" ]]; then - openapi-codegen generate \ - --template openapi-templates/go/api_resource.go.jinja \ - --input-file lib-openapi.json \ - --output-dir go - openapi-codegen generate \ - --template openapi-templates/go/component_type_summary.go.jinja \ - --input-file lib-openapi.json \ - --output-dir go - openapi-codegen generate \ - --template openapi-templates/go/component_type.go.jinja \ - --input-file lib-openapi.json \ - --output-dir go/models - rm go/{environment,health,ingest_endpoint}.go -fi - -# === Java === -if [[ -z "$CI" ]]; then - openapi-codegen generate \ - --template java/templates/api_resource.java.jinja \ - --input-file lib-openapi.json \ - --output-dir java/lib/src/main/java/com/svix/api - openapi-codegen generate \ - --template java/templates/operation_options.java.jinja \ - --input-file lib-openapi.json \ - --output-dir java/lib/src/main/java/com/svix/api - openapi-codegen generate \ - --template java/templates/component_type.java.jinja \ - --input-file lib-openapi.json \ - --output-dir java/lib/src/main/java/com/svix/models -fi - -# === C# === -if [[ -z "$CI" ]]; then - set -x - - openapi-codegen generate \ - --template csharp/templates/api_resource.cs.jinja \ - --input-file lib-openapi.json \ - --output-dir csharp/Svix - openapi-codegen generate \ - --template csharp/templates/component_type.cs.jinja \ - --input-file lib-openapi.json \ - --output-dir csharp/Svix/Models - - # Remove APIs we may not (yet) want to expose - rm csharp/Svix/IngestEndpoint.cs -fi - - -# === Ruby === -if [[ -z "$CI" ]]; then - openapi-codegen generate \ - --template ruby/templates/api_resource.rb.jinja \ - --input-file lib-openapi.json \ - --output-dir ruby/lib/svix/api - openapi-codegen generate \ - --template ruby/templates/summary.rb.jinja \ - --input-file lib-openapi.json \ - --output-dir ruby/lib - openapi-codegen generate \ - --template ruby/templates/component_type.rb.jinja \ - --input-file lib-openapi.json \ - --output-dir ruby/lib/svix/models -fi - - -# === JavaScript === -openapi-codegen generate \ - --template javascript/templates/api_resource.ts.jinja \ - --input-file lib-openapi.json \ - --output-dir javascript/src/api -openapi-codegen generate \ - --template javascript/templates/component_type_summary.ts.jinja \ - --input-file lib-openapi.json \ - --output-dir javascript/src/models -openapi-codegen generate \ - --template javascript/templates/component_type.ts.jinja \ - --input-file lib-openapi.json \ - --output-dir javascript/src/models - - -# === Rust === -openapi-codegen generate \ - --template rust/templates/api_resource.rs.jinja \ - --input-file lib-openapi.json \ - --output-dir rust/src/api -openapi-codegen generate \ - --template rust/templates/component_type.rs.jinja \ - --input-file lib-openapi.json \ - --output-dir rust/src/models - -# Remove APIs we may not (yet) want to expose -rm rust/src/api/{environment,health}.rs - -# === CLI === -openapi-codegen generate \ - --template svix-cli/templates/api_resource.rs.jinja \ - --input-file lib-openapi.json \ - --output-dir svix-cli/src/cmds/api - -# Our CLI templates currently output some unused imports. Get rid of them. -cargo fix --manifest-path svix-cli/Cargo.toml --allow-dirty -# `cargo fix` can leave the source in an inconsistently-formatted state. -cargo fmt --manifest-path svix-cli/Cargo.toml - -# Remove APIs we may not (yet) want to expose -rm svix-cli/src/cmds/api/{background_task,environment,health,operational_webhook_endpoint,statistics}.rs - -# === Python === - -#openapi-codegen generate \ -# --template python/templates/api_summary.py.jinja \ -# --input-file lib-openapi.json \ -# --output-dir python/svix/api -openapi-codegen generate \ - --template python/templates/api_resource.py.jinja \ - --input-file lib-openapi.json \ - --output-dir python/svix/api -openapi-codegen generate \ - --template python/templates/component_type_summary.py.jinja \ - --input-file lib-openapi.json \ - --output-dir python/svix/models -openapi-codegen generate \ - --template python/templates/component_type.py.jinja \ - --input-file lib-openapi.json \ - --output-dir python/svix/models - -# Remove APIs we may not (yet) want to expose -rm python/svix/api/{environment,health}.py