From 18de1d2bd745398cc81862594a9b46464aa81d97 Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Sat, 23 Dec 2023 03:35:12 +0000 Subject: [PATCH] feat(ports): `InstallsDb` --- host/mod.ts | 2 +- install/mod.ts | 2 +- mod.ts | 4 +- modules/ports/db.ts | 59 +++++ modules/ports/mod.ts | 39 ++- modules/ports/std.ts | 3 +- modules/ports/sync.ts | 579 +++++++++++++++++++++++++---------------- modules/ports/types.ts | 23 +- modules/std.ts | 35 +-- modules/tasks/mod.ts | 18 +- ports/asdf.ts | 5 +- tests/ports.ts | 10 +- utils/logger.ts | 23 +- utils/mod.ts | 9 +- 14 files changed, 527 insertions(+), 284 deletions(-) create mode 100644 modules/ports/db.ts diff --git a/host/mod.ts b/host/mod.ts index 4b0e9858..d2efc764 100644 --- a/host/mod.ts +++ b/host/mod.ts @@ -86,7 +86,7 @@ export async function cli(args: CliArgs) { if (!mod) { throw new Error(`unrecognized module specified by ghjk.ts: ${man.id}`); } - const instance = mod.ctor(ctx, man); + const instance = mod.init(ctx, man); cmd = cmd.command(man.id, instance.command()); } await cmd diff --git a/install/mod.ts b/install/mod.ts index 5a891f37..e0d377bc 100644 --- a/install/mod.ts +++ b/install/mod.ts @@ -197,7 +197,7 @@ export async function install( exePath, `#!/bin/sh GHJK_DIR="$\{GHJK_DIR:-${ghjkDir}}" DENO_DIR="$\{GHJK_DENO_DIR:-${denoCacheDir}}" -${args.ghjkExecDenoExec} run --unstable-worker-options -A ${lockFlag} ${ +${args.ghjkExecDenoExec} run --unstable-kv --unstable-worker-options -A ${lockFlag} ${ import.meta.resolve("../main.ts") } $*`, { mode: 0o700 }, diff --git a/mod.ts b/mod.ts index 2e631243..1e5433e1 100644 --- a/mod.ts +++ b/mod.ts @@ -15,7 +15,7 @@ import type { PortsModuleSecureConfig, } from "./modules/ports/types.ts"; import logger from "./utils/logger.ts"; -import { $ } from "./utils/mod.ts"; +import { $, getPortRef } from "./utils/mod.ts"; import * as std_ports from "./modules/ports/std.ts"; import * as cpy from "./ports/cpy_bs.ts"; import * as node from "./ports/node.ts"; @@ -108,7 +108,7 @@ function stdDeps(args = { enableRuntimes: false }) { return portsValidators.allowedPortDep.parse({ manifest: port, defaultInst: { - portName: port.name, + portRef: getPortRef(port), ...liteInst, }, }); diff --git a/modules/ports/db.ts b/modules/ports/db.ts new file mode 100644 index 00000000..1b8385ba --- /dev/null +++ b/modules/ports/db.ts @@ -0,0 +1,59 @@ +// Deno.Kv api is unstable +/// + +import type { + DownloadArtifacts, + InstallArtifacts, + InstallConfigLite, + PortManifestX, +} from "./types.ts"; + +export type InstallRow = { + installId: string; + conf: InstallConfigLite; + manifest: PortManifestX; + installArts?: InstallArtifacts; + downloadArts: DownloadArtifacts; + progress: "downloaded" | "installed"; +}; + +export abstract class InstallsDb { + abstract all(): Promise; + abstract get(id: string): Promise; + abstract set(id: string, row: InstallRow): Promise; + abstract delete(id: string): Promise; + abstract [Symbol.dispose](): void; +} + +export async function installsDbKv(path: string): Promise { + const kv = await Deno.openKv(path); + return new DenoKvInstallsDb(kv); +} + +class DenoKvInstallsDb extends InstallsDb { + prefix = "installs"; + constructor(public kv: Deno.Kv) { + super(); + } + async all(): Promise { + const iterator = this.kv.list({ prefix: [this.prefix] }); + return (await Array.fromAsync(iterator)).map((ent) => ent.value); + } + async get(id: string) { + const val = await this.kv.get([this.prefix, id]); + if (!val.value) return; + return val.value; + } + async set(id: string, row: InstallRow) { + const res = await this.kv.set([this.prefix, id], row); + if (!res.ok) { + throw new Error("Error on to Deno.Kv.set"); + } + } + async delete(id: string) { + await this.kv.delete([this.prefix, id]); + } + [Symbol.dispose](): void { + this.kv.close(); + } +} diff --git a/modules/ports/mod.ts b/modules/ports/mod.ts index 1fb23c33..4e78f324 100644 --- a/modules/ports/mod.ts +++ b/modules/ports/mod.ts @@ -1,16 +1,33 @@ export * from "./types.ts"; import { cliffy_cmd } from "../../deps/cli.ts"; - +import { $ } from "../../utils/mod.ts"; +import validators from "./types.ts"; import type { PortsModuleConfig } from "./types.ts"; -import type { GhjkCtx } from "../types.ts"; +import type { GhjkCtx, ModuleManifest } from "../types.ts"; import { ModuleBase } from "../mod.ts"; import { sync } from "./sync.ts"; +import { installsDbKv } from "./db.ts"; export class PortsModule extends ModuleBase { + public static init( + ctx: GhjkCtx, + manifest: ModuleManifest, + ) { + const res = validators.portsModuleConfig.safeParse(manifest.config); + if (!res.success) { + throw new Error("error parsing ports module config", { + cause: { + config: manifest.config, + zodErr: res.error, + }, + }); + } + return new PortsModule(ctx, res.data); + } constructor( - public ctx: GhjkCtx, - public config: PortsModuleConfig, + private ctx: GhjkCtx, + private config: PortsModuleConfig, ) { super(); } @@ -24,7 +41,19 @@ export class PortsModule extends ModuleBase { .command( "sync", new cliffy_cmd.Command().description("Syncs the environment.") - .action(() => sync(this.ctx.ghjkDir, this.ctx.envDir, this.config)), + .action(async () => { + const portsDir = await $.path(this.ctx.ghjkDir).resolve("ports") + .ensureDir(); + using db = await installsDbKv( + portsDir.resolve("installs.db").toString(), + ); + return await sync( + portsDir.toString(), + this.ctx.envDir, + this.config, + db, + ); + }), ) .command( "outdated", diff --git a/modules/ports/std.ts b/modules/ports/std.ts index fa39958e..a2e54a75 100644 --- a/modules/ports/std.ts +++ b/modules/ports/std.ts @@ -15,6 +15,7 @@ import { manifest as man_node_org } from "../../ports/node.ts"; import { manifest as man_pnpm_ghrel } from "../../ports/pnpm.ts"; import { manifest as man_asdf_plugin_git } from "../../ports/asdf_plugin_git.ts"; import { manifest as man_cpy_bs_ghrel } from "../../ports/cpy_bs.ts"; +import { getPortRef } from "../../utils/mod.ts"; const aaPorts: PortManifest[] = [ man_tar_aa, @@ -39,7 +40,7 @@ const defaultAllowedDeps: AllowedPortDepX[] = [ .map((manifest) => ({ manifest, defaultInst: { - portName: manifest.name, + portRef: getPortRef(manifest), }, })) .map((portDep) => validators.allowedPortDep.parse(portDep)); diff --git a/modules/ports/sync.ts b/modules/ports/sync.ts index 14b49f60..50074005 100644 --- a/modules/ports/sync.ts +++ b/modules/ports/sync.ts @@ -1,9 +1,12 @@ import { equal, std_fs, std_path, zod } from "../../deps/cli.ts"; -import logger from "../../utils/logger.ts"; -import validators, { +import getLogger from "../../utils/logger.ts"; +import validators from "./types.ts"; +import type { AmbientAccessPortManifestX, DenoWorkerPortManifestX, DepArts, + DownloadArtifacts, + InstallArtifacts, InstallConfigLite, InstallConfigLiteX, PortArgsBase, @@ -12,182 +15,113 @@ import validators, { } from "./types.ts"; import { DenoWorkerPort } from "./worker.ts"; import { AmbientAccessPort } from "./ambient.ts"; -import { $, AVAIL_CONCURRENCY, getInstallHash } from "../../utils/mod.ts"; +import { + $, + AVAIL_CONCURRENCY, + getInstallHash, + getPortRef, +} from "../../utils/mod.ts"; +import type { InstallsDb } from "./db.ts"; -// create the loader scripts -// loader scripts are responsible for exporting -// different environment variables from the ports -// and mainpulating the path strings -async function writeLoader( - envDir: string, - env: Record, - pathVars: Record, -) { - const loader = { - posix: [ - `export GHJK_CLEANUP_POSIX="";`, - ...Object.entries(env).map(([k, v]) => - // NOTE: single quote the port supplied envs to avoid any embedded expansion/execution - `GHJK_CLEANUP_POSIX=$GHJK_CLEANUP_POSIX"export ${k}='$${k}';"; -export ${k}='${v}';` - ), - ...Object.entries(pathVars).map(([k, v]) => - // NOTE: double quote the path vars for expansion - // single quote GHJK_CLEANUP additions to avoid expansion/exec before eval - `GHJK_CLEANUP_POSIX=$GHJK_CLEANUP_POSIX'${k}=$(echo "$${k}" | tr ":" "\\n" | grep -vE "^${envDir}" | tr "\\n" ":");${k}="\${${k}%:}"'; -export ${k}="${v}:$${k}"; -` - ), - ].join("\n"), - fish: [ - `set --erase GHJK_CLEANUP_FISH`, - ...Object.entries(env).map(([k, v]) => - `set --global --append GHJK_CLEANUP_FISH "set --global --export ${k} '$${k}';"; -set --global --export ${k} '${v}';` - ), - ...Object.entries(pathVars).map(([k, v]) => - `set --global --append GHJK_CLEANUP_FISH 'set --global --path ${k} (string match --invert --regex "^${envDir}" $${k});'; -set --global --prepend ${k} ${v}; -` - ), - ].join("\n"), - }; - const envPathR = await $.path(envDir).ensureDir(); - await Promise.all([ - envPathR.join(`loader.fish`).writeText(loader.fish), - envPathR.join(`loader.sh`).writeText(loader.posix), - ]); -} - -/// this returns a tmp path that's guaranteed to be -/// on the same file system as targetDir by -/// checking if $TMPDIR satisfies that constraint -/// or just pointing to targetDir/tmp -/// This is handy for making moves atomics from -/// tmp dirs to to locations within targetDir -async function movebleTmpPath(targetDir: string, targetTmpDirName = "dir") { - const defaultTmp = Deno.env.get("TMPDIR"); - const targetPath = $.path(targetDir); - if (!defaultTmp) { - return targetPath.join(targetTmpDirName); - } - const defaultTmpPath = $.path(defaultTmp); - if ((await targetPath.stat())?.dev != (await defaultTmpPath.stat())?.dev) { - return targetPath.join(targetTmpDirName); - } - return defaultTmpPath.join("ghjkTmp"); -} +const logger = getLogger(import.meta); export async function sync( - ghjkDir: string, + portsDir: string, envDir: string, cx: PortsModuleConfig, + installsDb: InstallsDb, ) { - const ghjkPathR = $.path(ghjkDir); + logger.debug("syncing ports"); + const portsPath = $.path(portsDir); // ensure the req const [installsPath, downloadsPath, tmpPath] = ( await Promise.all([ - ghjkPathR.join("ports", "installs").ensureDir(), - ghjkPathR.join("ports", "downloads").ensureDir(), - (await movebleTmpPath(ghjkDir)).ensureDir(), + portsPath.join("installs").ensureDir(), + portsPath.join("downloads").ensureDir(), + (await movebleTmpPath(portsDir)).ensureDir(), ]) ).map($.pathToString); const graph = await buildInstallGraph(cx); - const artifacts = new Map(); // start from the ports with no build deps const pendingInstalls = [...graph.indie]; - // deep clone graph.depEdges for a list of deps for each port - // to tick of as we work through the graph - // initial graph.depEdges is needed intact for other purposes - const pendingDepEdges = new Map( - [...graph.depEdges.entries()].map(([key, val]) => [key, [...val]]), - ); while (pendingInstalls.length > 0) { const installId = pendingInstalls.pop()!; - const inst = graph.all.get(installId)!; + const cached = await installsDb.get(installId); - const manifest = graph.ports.get(inst.portRef)!; + let thisArtifacts; + // we skip it if it's already installed + if (cached && cached.progress == "installed") { + logger.debug("already installed, skipping", installId); + thisArtifacts = cached.installArts!; + } else { + const inst = graph.all.get(installId)!; - // make an object detailing all the artifacts - // that the port's deps exported - const totalDepArts: DepArts = {}; - const depShimsRootPath = await Deno.makeTempDir({ - dir: tmpPath, - prefix: `shims_${installId}_`, - }); - for ( - const [depInstallId, depPortName] of graph.depEdges.get(installId) ?? [] - ) { - const depArts = artifacts.get(depInstallId); - if (!depArts) { - throw new Error( - `artifacts not found for plug dep "${depInstallId}" when installing "${installId}"`, - ); - } - const depShimDir = $.path(depShimsRootPath).resolve(depInstallId); - const [binShimDir, libShimDir, includeShimDir] = (await Promise.all([ - depShimDir.join("bin").ensureDir(), - depShimDir.join("lib").ensureDir(), - depShimDir.join("include").ensureDir(), - ])).map($.pathToString); - - totalDepArts[depPortName] = { - execs: await shimLinkPaths( - depArts.binPaths, - depArts.installPath, - binShimDir, - ), - libs: await shimLinkPaths( - depArts.libPaths, - depArts.installPath, - libShimDir, - ), - includes: await shimLinkPaths( - depArts.includePaths, - depArts.installPath, - includeShimDir, - ), - env: depArts.env, - }; - } + const manifest = graph.ports.get(inst.portRef)!; - let thisArtifacts; - try { - thisArtifacts = await doInstall( - installsPath, - downloadsPath, + // readys all the exports of the port's deps including + // shims for their exports + const { totalDepArts, depShimsRootPath } = await graph.readyDepArts( tmpPath, - inst.conf, - manifest, - totalDepArts, + installId, ); - } catch (err) { - throw new Error(`error installing ${installId}`, { cause: err }); - } - artifacts.set(installId, thisArtifacts); - void $.removeIfExists(depShimsRootPath); - - // mark where appropriate if some other install was waiting on - // the current install - const parents = graph.revDepEdges.get(installId) ?? []; - for (const parentId of parents) { - const parentDeps = pendingDepEdges.get(parentId)!; + const stageArgs = { + installId, + installPath: std_path.resolve(installsPath, installId), + downloadPath: std_path.resolve(downloadsPath, installId), + tmpPath, + conf: inst.conf, + manifest, + depArts: totalDepArts, + }; - // swap remove from parent pending deps list - const idx = parentDeps.findIndex(([instId, _]) => instId == installId); - const last = parentDeps.pop()!; - if (parentDeps.length > idx) { - parentDeps[idx] = last; + const dbRow = { + installId, + conf: inst.conf, + manifest, + }; + let downloadArts; + if (cached) { + logger.debug("already downloaded, skipping to install", installId); + // download step must have completed if there's a cache hit + downloadArts = cached.downloadArts; + } else { + try { + downloadArts = await doDownloadStage({ + ...stageArgs, + }); + } catch (err) { + throw new Error(`error downloading ${installId}`, { cause: err }); + } + await installsDb.set(installId, { + ...dbRow, + progress: "downloaded", + downloadArts, + }); } - if (parentDeps.length == 0) { - // parent is ready for install - pendingInstalls.push(parentId); + try { + thisArtifacts = await doInstallStage( + { + ...stageArgs, + ...downloadArts, + }, + ); + } catch (err) { + throw new Error(`error installing ${installId}`, { cause: err }); } + await installsDb.set(installId, { + ...dbRow, + progress: "installed", + downloadArts, + installArts: thisArtifacts, + }); + void $.removeIfExists(depShimsRootPath); } + graph.artifacts.set(installId, thisArtifacts); + pendingInstalls.push(...graph.installDone(installId)); } // create the shims for the user's environment @@ -203,9 +137,10 @@ export async function sync( // FIXME: detect conflicts // FIXME: better support for multi installs for (const instId of graph.user) { - const { binPaths, libPaths, includePaths, installPath } = artifacts.get( - instId, - )!; + const { binPaths, libPaths, includePaths, installPath } = graph.artifacts + .get( + instId, + )!; // bin shims void await shimLinkPaths( binPaths, @@ -228,7 +163,7 @@ export async function sync( // write loader for the env vars mandated by the installs const env: Record = {}; - for (const [instId, item] of artifacts) { + for (const [instId, item] of graph.artifacts) { for (const [key, val] of Object.entries(item.env)) { const conflict = env[key]; if (conflict) { @@ -259,7 +194,7 @@ export async function sync( C_INCLUDE_PATH: `${envDir}/shims/include`, CPLUS_INCLUDE_PATH: `${envDir}/shims/include`, }; - logger().debug("adding vars to loader", env); + logger.debug("adding vars to loader", env); // FIXME: prevent malicious env manipulations await writeLoader( envDir, @@ -271,6 +206,25 @@ export async function sync( await $.removeIfExists(tmpPath); } +/// this returns a tmp path that's guaranteed to be +/// on the same file system as targetDir by +/// checking if $TMPDIR satisfies that constraint +/// or just pointing to targetDir/tmp +/// This is handy for making moves atomics from +/// tmp dirs to to locations within targetDir +async function movebleTmpPath(targetDir: string, targetTmpDirName = "tmp") { + const defaultTmp = Deno.env.get("TMPDIR"); + const targetPath = $.path(targetDir); + if (!defaultTmp) { + return targetPath.join(targetTmpDirName); + } + const defaultTmpPath = $.path(defaultTmp); + if ((await targetPath.stat())?.dev != (await defaultTmpPath.stat())?.dev) { + return targetPath.join(targetTmpDirName); + } + return defaultTmpPath.join("portsTmp"); +} + // this returns a data structure containing all the info // required for installation including the dependency graph async function buildInstallGraph(cx: PortsModuleConfig) { @@ -295,36 +249,115 @@ async function buildInstallGraph(cx: PortsModuleConfig) { depEdges: new Map(), // the manifests of the ports ports: new Map(), - }; - // add port to ports list - const addPort = (manifest: PortManifestX) => { - const portRef = `${manifest.name}@${manifest.version}`; + // the end artifacts of a port + artifacts: new Map(), + // a deep clone graph.depEdges for a list of deps for each port + // to tick of as we work through the graph + // initial graph.depEdges is needed intact for other purposes + pendingDepEdges: new Map(), + addPort(manifest: PortManifestX) { + const portRef = `${manifest.name}@${manifest.version}`; + + const conflict = graph.ports.get(portRef); + if (conflict) { + if (!equal.equal(conflict, manifest)) { + throw new Error( + `differing port manifests found for "${portRef}: ${ + $.inspect(manifest) + }" != ${$.inspect(conflict)}`, + ); + } + } else { + graph.ports.set(portRef, manifest); + } - const conflict = graph.ports.get(portRef); - if (conflict) { - if (!equal.equal(conflict, manifest)) { - throw new Error( - `differing port manifests found for "${portRef}: ${ - $.inspect(manifest) - }" != ${$.inspect(conflict)}`, - ); + return portRef; + }, + + // make an object detailing all the artifacts + // that the port's deps have exported + async readyDepArts( + tmpPath: string, + installId: string, + ) { + const totalDepArts: DepArts = {}; + const depShimsRootPath = await Deno.makeTempDir({ + dir: tmpPath, + prefix: `shims_${installId}_`, + }); + for ( + const [depInstallId, depPortName] of graph.depEdges.get(installId) ?? [] + ) { + const depArts = graph.artifacts.get(depInstallId); + if (!depArts) { + throw new Error( + `artifacts not found for plug dep "${depInstallId}" when installing "${installId}"`, + ); + } + const depShimDir = $.path(depShimsRootPath).resolve(depInstallId); + const [binShimDir, libShimDir, includeShimDir] = (await Promise.all([ + depShimDir.join("bin").ensureDir(), + depShimDir.join("lib").ensureDir(), + depShimDir.join("include").ensureDir(), + ])).map($.pathToString); + + totalDepArts[depPortName] = { + execs: await shimLinkPaths( + depArts.binPaths, + depArts.installPath, + binShimDir, + ), + libs: await shimLinkPaths( + depArts.libPaths, + depArts.installPath, + libShimDir, + ), + includes: await shimLinkPaths( + depArts.includePaths, + depArts.installPath, + includeShimDir, + ), + env: depArts.env, + }; } - } else { - graph.ports.set(portRef, manifest); - } + return { totalDepArts, depShimsRootPath }; + }, + installDone(installId: string) { + // mark where appropriate if some other install was waiting on + // the current install + const parents = graph.revDepEdges.get(installId) ?? []; + // list of parents that are ready for installation now + // that their dep is fullfilled + const readyParents = []; + for (const parentId of parents) { + const parentDeps = graph.pendingDepEdges.get(parentId)!; + + // swap remove from parent pending deps list + const idx = parentDeps.findIndex(([instId, _]) => instId == installId); + const last = parentDeps.pop()!; + if (parentDeps.length > idx) { + parentDeps[idx] = last; + } - return portRef; + if (parentDeps.length == 0) { + // parent is ready for install + readyParents.push(parentId); + } + } + return readyParents; + }, }; + // add port to ports list const foundInstalls: GraphInstConf[] = []; // collect the user specified insts first for (const inst of cx.installs) { const { port, ...instLiteBase } = inst; - const portRef = addPort(port); + const portRef = graph.addPort(port); const instLite = validators.installConfigLite.parse({ ...instLiteBase, - portName: port.name, + portRef: getPortRef(port), }); const instId = await getInstallHash(instLite); @@ -409,7 +442,7 @@ async function buildInstallGraph(cx: PortsModuleConfig) { // only add the install configuration for this dep port // if specific hash hasn't seen before if (!graph.all.has(depInstallId)) { - const portRef = addPort(depPort); + const portRef = graph.addPort(depPort); foundInstalls.push({ conf: depInstall, portRef }); } @@ -424,6 +457,9 @@ async function buildInstallGraph(cx: PortsModuleConfig) { } } + graph.pendingDepEdges = new Map( + [...graph.depEdges.entries()].map(([key, val]) => [key, [...val]]), + ); return graph; } @@ -471,36 +507,14 @@ async function shimLinkPaths( return shims; } -type DePromisify = T extends Promise ? Inner : T; -type InstallArtifacts = DePromisify>; - -/// Drive a port implementation so that it does its thing -async function doInstall( - installsDir: string, - downloadsDir: string, - tmpDir: string, - instUnclean: InstallConfigLite, - manifest: PortManifestX, - depArts: DepArts, -) { - logger().debug("installing", { - installsDir, - downloadsDir, - instUnclean, - port: manifest, - }); - - // instantiate the right Port impl according to manifest.ty - let port; - let inst: InstallConfigLiteX; +// instantiates the right Port impl according to manifest.ty +function getPortImpl(manifest: PortManifestX) { if (manifest.ty == "denoWorker@v1") { - inst = validators.installConfigLite.parse(instUnclean); - port = new DenoWorkerPort( + return new DenoWorkerPort( manifest as DenoWorkerPortManifestX, ); } else if (manifest.ty == "ambientAccess@v1") { - inst = validators.installConfigLite.parse(instUnclean); - port = new AmbientAccessPort( + return new AmbientAccessPort( manifest as AmbientAccessPortManifestX, ); } else { @@ -510,43 +524,113 @@ async function doInstall( }`, ); } +} + +async function doDownloadStage( + { + installId, + installPath, + downloadPath, + tmpPath, + conf, + manifest, + depArts, + }: { + installId: string; + installPath: string; + downloadPath: string; + tmpPath: string; + conf: InstallConfigLiteX; + manifest: PortManifestX; + depArts: DepArts; + }, +) { + logger.debug("downloading", { + installId, + installPath, + downloadPath, + conf, + port: manifest, + }); + + const port = getPortImpl(manifest); - const installId = await getInstallHash(inst); const installVersion = validators.string.parse( - inst.version ?? + conf.version ?? await port.latestStable({ depArts, manifest, - config: inst, + config: conf, }), ); - const installPath = std_path.resolve(installsDir, installId); - const downloadPath = std_path.resolve(downloadsDir, installId); - const baseArgs: PortArgsBase = { + + logger.info(`downloading ${installId}:${installVersion}`); + const tmpDirPath = await Deno.makeTempDir({ + dir: tmpPath, + prefix: `download_${installId}@${installVersion}_`, + }); + await port.download({ installPath: installPath, installVersion: installVersion, depArts, platform: Deno.build, - config: inst, + config: conf, manifest, + downloadPath, + tmpDirPath, + }); + void $.removeIfExists(tmpDirPath); + + const out: DownloadArtifacts = { + downloadPath, + installVersion, }; + return out; +} + +async function doInstallStage( { - logger().info(`downloading ${installId}:${installVersion}`); - const tmpDirPath = await Deno.makeTempDir({ - dir: tmpDir, - prefix: `download_${installId}@${installVersion}_`, - }); - await port.download({ - ...baseArgs, - downloadPath: downloadPath, - tmpDirPath, - }); - void $.removeIfExists(tmpDirPath); - } + installId, + installPath, + downloadPath, + tmpPath, + conf, + manifest, + depArts, + installVersion, + }: { + installId: string; + installPath: string; + downloadPath: string; + tmpPath: string; + conf: InstallConfigLite; + manifest: PortManifestX; + depArts: DepArts; + installVersion: string; + }, +) { + logger.debug("installing", { + installId, + installPath, + downloadPath, + conf, + port: manifest, + }); + + const port = getPortImpl(manifest); + + const baseArgs: PortArgsBase = { + installPath, + installVersion, + depArts, + platform: Deno.build, + config: conf, + manifest, + }; { - logger().info(`installing ${installId}:${installVersion}`); + logger.info(`installing ${installId}:${installVersion}`); const tmpDirPath = await Deno.makeTempDir({ - dir: tmpDir, + dir: tmpPath, prefix: `install_${installId}@${installVersion}_`, }); await port.install({ @@ -577,5 +661,58 @@ async function doInstall( ...baseArgs, }), ); - return { env, binPaths, libPaths, includePaths, installPath, downloadPath }; + return { + env, + binPaths, + libPaths, + includePaths, + installPath, + downloadPath, + installVersion, + }; +} + +// create the loader scripts +// loader scripts are responsible for exporting +// different environment variables from the ports +// and mainpulating the path strings +async function writeLoader( + envDir: string, + env: Record, + pathVars: Record, +) { + const loader = { + posix: [ + `export GHJK_CLEANUP_POSIX="";`, + ...Object.entries(env).map(([k, v]) => + // NOTE: single quote the port supplied envs to avoid any embedded expansion/execution + `GHJK_CLEANUP_POSIX=$GHJK_CLEANUP_POSIX"export ${k}='$${k}';"; +export ${k}='${v}';` + ), + ...Object.entries(pathVars).map(([k, v]) => + // NOTE: double quote the path vars for expansion + // single quote GHJK_CLEANUP additions to avoid expansion/exec before eval + `GHJK_CLEANUP_POSIX=$GHJK_CLEANUP_POSIX'${k}=$(echo "$${k}" | tr ":" "\\n" | grep -vE "^${envDir}" | tr "\\n" ":");${k}="\${${k}%:}"'; +export ${k}="${v}:$${k}"; +` + ), + ].join("\n"), + fish: [ + `set --erase GHJK_CLEANUP_FISH`, + ...Object.entries(env).map(([k, v]) => + `set --global --append GHJK_CLEANUP_FISH "set --global --export ${k} '$${k}';"; +set --global --export ${k} '${v}';` + ), + ...Object.entries(pathVars).map(([k, v]) => + `set --global --append GHJK_CLEANUP_FISH 'set --global --path ${k} (string match --invert --regex "^${envDir}" $${k});'; +set --global --prepend ${k} ${v}; +` + ), + ].join("\n"), + }; + const envPathR = await $.path(envDir).ensureDir(); + await Promise.all([ + envPathR.join(`loader.fish`).writeText(loader.fish), + envPathR.join(`loader.sh`).writeText(loader.posix), + ]); } diff --git a/modules/ports/types.ts b/modules/ports/types.ts index 8bf8717b..7e000392 100644 --- a/modules/ports/types.ts +++ b/modules/ports/types.ts @@ -4,7 +4,11 @@ import { semver, zod } from "../../deps/common.ts"; // TODO: find a better identification scheme for ports const portName = zod.string().regex(/[^ @]*/); -// const portRef = zod.string().regex(/[^ ]*@[^ ]/); +// FIXME: get rid of semantic minor.patch version from portRef +// to allow install hashes to be equivalent as long as major +// version is the same +// Or alternatively, drop semnatic versioning ports +const portRef = zod.string().regex(/[^ ]*@\d+\.\d+\.\d+/); const portDep = zod.object({ name: portName, @@ -100,7 +104,7 @@ const installConfigBaseFat = installConfigBase.merge(zod.object({ })).passthrough(); const installConfigBaseLite = installConfigBase.merge(zod.object({ - portName: portName, + portRef, })).passthrough(); const stdInstallConfigFat = installConfigBaseFat.merge(zod.object({})) @@ -310,3 +314,18 @@ export interface InstallArgs extends PortArgsBase { downloadPath: string; tmpDirPath: string; } + +export type DownloadArtifacts = { + installVersion: string; + downloadPath: string; +}; + +export type InstallArtifacts = { + env: Record; + installVersion: string; + binPaths: string[]; + libPaths: string[]; + includePaths: string[]; + installPath: string; + downloadPath: string; +}; diff --git a/modules/std.ts b/modules/std.ts index 5edf3788..8f170160 100644 --- a/modules/std.ts +++ b/modules/std.ts @@ -1,8 +1,5 @@ import { PortsModule } from "./ports/mod.ts"; -import portsValidators from "./ports/types.ts"; import { TasksModule } from "./tasks/mod.ts"; -import tasksValidators from "./tasks/types.ts"; -import type { GhjkCtx, ModuleManifest } from "./types.ts"; export const ports = "ports"; @@ -10,37 +7,9 @@ export const tasks = "tasks"; export const map = { [ports as string]: { - ctor: (ctx: GhjkCtx, manifest: ModuleManifest) => { - const res = portsValidators.portsModuleConfig.safeParse(manifest.config); - if (!res.success) { - throw new Error("error parsing ports module config", { - cause: { - config: manifest.config, - zodErr: res.error, - }, - }); - } - return new PortsModule( - ctx, - res.data, - ); - }, + init: PortsModule.init, }, [tasks as string]: { - ctor: (ctx: GhjkCtx, manifest: ModuleManifest) => { - const res = tasksValidators.tasksModuleConfig.safeParse(manifest.config); - if (!res.success) { - throw new Error("error parsing tasks module config", { - cause: { - config: manifest.config, - zodErr: res.error, - }, - }); - } - return new TasksModule( - ctx, - res.data, - ); - }, + init: TasksModule.init, }, }; diff --git a/modules/tasks/mod.ts b/modules/tasks/mod.ts index 45130607..066cae9e 100644 --- a/modules/tasks/mod.ts +++ b/modules/tasks/mod.ts @@ -23,13 +23,29 @@ export * from "./types.ts"; import { cliffy_cmd, std_path } from "../../deps/cli.ts"; +import validators from "./types.ts"; import type { TasksModuleConfig } from "./types.ts"; -import type { GhjkCtx } from "../types.ts"; +import type { GhjkCtx, ModuleManifest } from "../types.ts"; import { ModuleBase } from "../mod.ts"; import logger from "../../utils/logger.ts"; import { execTaskDeno } from "./deno.ts"; export class TasksModule extends ModuleBase { + public static init( + ctx: GhjkCtx, + manifest: ModuleManifest, + ) { + const res = validators.tasksModuleConfig.safeParse(manifest.config); + if (!res.success) { + throw new Error("error parsing ports module config", { + cause: { + config: manifest.config, + zodErr: res.error, + }, + }); + } + return new TasksModule(ctx, res.data); + } constructor( public ctx: GhjkCtx, public config: TasksModuleConfig, diff --git a/ports/asdf.ts b/ports/asdf.ts index 9968b5af..2ce0012a 100644 --- a/ports/asdf.ts +++ b/ports/asdf.ts @@ -2,6 +2,7 @@ import { $, depExecShimPath, DownloadArgs, + getPortRef, InstallArgs, InstallConfigFat, InstallConfigSimple, @@ -39,7 +40,7 @@ export default function conf( config: AsdfInstallConf, ): InstallConfigFat { // we only need the lite version of the InstConf here - const { port, ...liteConf } = asdf_plugin_git({ + const { port: pluginPort, ...liteConf } = asdf_plugin_git({ pluginRepo: config.pluginRepo, }); return { @@ -48,7 +49,7 @@ export default function conf( depConfigs: { [std_ports.asdf_plugin_git.name]: { ...liteConf, - portName: port.name, + portRef: getPortRef(pluginPort), }, }, }; diff --git a/tests/ports.ts b/tests/ports.ts index 1fe4a4c5..685e810c 100644 --- a/tests/ports.ts +++ b/tests/ports.ts @@ -152,9 +152,13 @@ function testMany( () => testFn({ ...testCase, - ePoints: ["bash -c", "fish -c", "zsh -c"].map((sh) => - `env ${sh} '${testCase.ePoint}'` - ), + ePoints: [ + ...["bash -c", "fish -c", "zsh -c"].map((sh) => + `env ${sh} '${testCase.ePoint}'` + ), + // installs db means this shouldn't take too log + "env bash -c 'timeout 1 ghjk ports sync'", + ], envs: { ...defaultEnvs, ...testCase.envs, diff --git a/utils/logger.ts b/utils/logger.ts index e6cd4b43..2c7f0582 100644 --- a/utils/logger.ts +++ b/utils/logger.ts @@ -1,12 +1,16 @@ -import { log, std_fmt_colors, std_path, std_url, zod } from "../deps/common.ts"; +import { log, std_fmt_colors, std_url, zod } from "../deps/common.ts"; -const defaultLogLevel = "INFO" as const; +const defaultLogLevel = "DEBUG" as const; // This parses the GHJK_LOG env var function confFromEnv() { + const loggerConfs = { "": defaultLogLevel } as Record< + string, + zod.infer + >; const confStr = Deno.env.get("GHJK_LOG"); if (!confStr) { - return { "": defaultLogLevel }; + return loggerConfs; } const levelValidator = zod.enum( ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], @@ -17,7 +21,6 @@ function confFromEnv() { let defaultLevel = levelValidator.parse(defaultLogLevel); const confs = confStr.toUpperCase().split(","); // configure specific named loggers - const loggerConfs = {} as Record>; for (const confSection of confs) { const [left, right] = confSection.split("="); // this is a plain level name, thus configuring the default logger @@ -87,7 +90,7 @@ export class ConsoleErrHandler extends log.handlers.BaseHandler { } const loggers = new Map(); -let loggerLevelsConf = confFromEnv(); +const loggerLevelsConf = confFromEnv(); const consoleHandler = new ConsoleErrHandler("NOTSET"); @@ -96,8 +99,9 @@ export default function logger( name: ImportMeta | string = self.name, ) { if (typeof name === "object") { - name = std_url.basename(name.url); - name = name.replace(std_path.extname(name), ""); + const baseName = std_url.basename(name.url); + const dirName = std_url.basename(std_url.dirname(name.url)); + name = `${dirName}/${baseName}`; } let logger = loggers.get(name); if (!logger) { @@ -110,11 +114,8 @@ export default function logger( } export function setup() { - loggerLevelsConf = { - ...loggerLevelsConf, - }; const defaultLogger = new log.Logger("default", loggerLevelsConf[""], { - handlers: [], + handlers: [consoleHandler], }); loggers.set("", defaultLogger); } diff --git a/utils/mod.ts b/utils/mod.ts index ae264b09..d6642d7d 100644 --- a/utils/mod.ts +++ b/utils/mod.ts @@ -7,8 +7,11 @@ import type { InstallConfigLite, OsEnum, PortDep, + PortManifest, } from "../modules/ports/types.ts"; +export type DePromisify = T extends Promise ? Inner : T; + export function dbg(val: T, ...more: unknown[]) { logger().debug(() => val, ...more); return val; @@ -108,10 +111,14 @@ export function bufferToHex(buffer: ArrayBuffer): string { ).join(""); } +export function getPortRef(manifest: PortManifest) { + return `${manifest.name}@${manifest.version}`; +} + export async function getInstallHash(install: InstallConfigLite) { const hashBuf = await jsonHash.digest("SHA-256", install as jsonHash.Tree); const hashHex = bufferToHex(hashBuf).slice(0, 8); - return `${install.portName}@${hashHex}`; + return `${install.portRef}+${hashHex}`; } export const $ = dax.build$(