diff --git a/ghjk.lock b/ghjk.lock index 965d4878..d788162f 100644 --- a/ghjk.lock +++ b/ghjk.lock @@ -37,7 +37,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/act.ts" } }, @@ -112,7 +111,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "cpy_bs_ghrel" @@ -153,7 +151,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "tar_aa" @@ -190,7 +187,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "tar", "versionExtractFlag": "--version", "versionExtractRegex": "(\\d+\\.\\d+)", @@ -271,7 +267,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "git", "versionExtractFlag": "--version", "versionExtractRegex": "(\\d+\\.\\d+\\.\\d+)", @@ -352,7 +347,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "curl", "versionExtractFlag": "--version", "versionExtractRegex": "(\\d+\\.\\d+\\.\\d+)", @@ -393,7 +387,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "unzip", "versionExtractFlag": "-v", "versionExtractRegex": "(\\d+\\.\\d+)", @@ -426,7 +419,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "zstd", "versionExtractFlag": "--version", "versionExtractRegex": "v(\\d+\\.\\d+\\.\\d+),", @@ -459,7 +451,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/cargo-binstall.ts" }, "defaultInst": { @@ -497,7 +488,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/pnpm.ts" }, "defaultInst": { @@ -535,12 +525,16 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "git_aa" } ], + "resolutionDeps": [ + { + "name": "git_aa" + } + ], "moduleSpecifier": "file:///data/home/ghjk/ports/asdf_plugin_git.ts" }, "defaultInst": { @@ -578,7 +572,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "tar_aa" @@ -621,7 +614,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "tar_aa" @@ -752,7 +744,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/act.ts" } }, @@ -827,7 +818,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "cpy_bs_ghrel" @@ -868,7 +858,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "tar_aa" @@ -905,7 +894,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "tar", "versionExtractFlag": "--version", "versionExtractRegex": "(\\d+\\.\\d+)", @@ -986,7 +974,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "git", "versionExtractFlag": "--version", "versionExtractRegex": "(\\d+\\.\\d+\\.\\d+)", @@ -1067,7 +1054,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "curl", "versionExtractFlag": "--version", "versionExtractRegex": "(\\d+\\.\\d+\\.\\d+)", @@ -1108,7 +1094,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "unzip", "versionExtractFlag": "-v", "versionExtractRegex": "(\\d+\\.\\d+)", @@ -1141,7 +1126,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "zstd", "versionExtractFlag": "--version", "versionExtractRegex": "v(\\d+\\.\\d+\\.\\d+),", @@ -1174,7 +1158,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/cargo-binstall.ts" }, "defaultInst": { @@ -1212,7 +1195,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/pnpm.ts" }, "defaultInst": { @@ -1250,12 +1232,16 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "git_aa" } ], + "resolutionDeps": [ + { + "name": "git_aa" + } + ], "moduleSpecifier": "file:///data/home/ghjk/ports/asdf_plugin_git.ts" }, "defaultInst": { @@ -1293,7 +1279,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "tar_aa" @@ -1336,7 +1321,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "tar_aa" @@ -1355,74 +1339,117 @@ }, "graph": { "all": { - "cpy_bs_ghrel@0.1.0+84ecde63": { + "cpy_bs_ghrel@0.1.0+5e64e9e2": { + "instId": "cpy_bs_ghrel@0.1.0+5e64e9e2", "portRef": "cpy_bs_ghrel@0.1.0", - "conf": { + "config": { + "version": "3.12.0", + "depConfigs": { + "tar_aa": { + "version": "1.34", + "depConfigs": {}, + "portRef": "tar_aa@0.1.0" + }, + "zstd_aa": { + "version": "v1.5.5,", + "depConfigs": {}, + "portRef": "zstd_aa@0.1.0" + } + }, "portRef": "cpy_bs_ghrel@0.1.0" } }, - "zstd_aa@0.1.0+4f16c720": { - "conf": { + "zstd_aa@0.1.0+993fa832": { + "config": { + "version": "v1.5.5,", + "depConfigs": {}, "portRef": "zstd_aa@0.1.0" }, - "portRef": "zstd_aa@0.1.0" + "portRef": "zstd_aa@0.1.0", + "instId": "zstd_aa@0.1.0+993fa832" }, - "tar_aa@0.1.0+9e3fa774": { - "conf": { + "tar_aa@0.1.0+d9cbe4e3": { + "config": { + "version": "1.34", + "depConfigs": {}, "portRef": "tar_aa@0.1.0" }, - "portRef": "tar_aa@0.1.0" + "portRef": "tar_aa@0.1.0", + "instId": "tar_aa@0.1.0+d9cbe4e3" }, - "pipi_pypi@0.1.0+076a5b8e": { + "pipi_pypi@0.1.0+94a90a13": { + "instId": "pipi_pypi@0.1.0+94a90a13", "portRef": "pipi_pypi@0.1.0", - "conf": { + "config": { + "version": "3.6.0", + "depConfigs": { + "cpy_bs_ghrel": { + "version": "3.12.0", + "depConfigs": { + "tar_aa": { + "version": "1.34", + "depConfigs": {}, + "portRef": "tar_aa@0.1.0" + }, + "zstd_aa": { + "version": "v1.5.5,", + "depConfigs": {}, + "portRef": "zstd_aa@0.1.0" + } + }, + "portRef": "cpy_bs_ghrel@0.1.0" + } + }, "portRef": "pipi_pypi@0.1.0", "packageName": "pre-commit" } }, - "act_ghrel@0.1.0+95dbc2b8": { + "act_ghrel@0.1.0+a57a28a0": { + "instId": "act_ghrel@0.1.0+a57a28a0", "portRef": "act_ghrel@0.1.0", - "conf": { + "config": { + "version": "v0.2.56", + "depConfigs": {}, "portRef": "act_ghrel@0.1.0" } } }, "user": [ - "act_ghrel@0.1.0+95dbc2b8", - "pipi_pypi@0.1.0+076a5b8e", - "cpy_bs_ghrel@0.1.0+84ecde63" + "act_ghrel@0.1.0+a57a28a0", + "pipi_pypi@0.1.0+94a90a13", + "cpy_bs_ghrel@0.1.0+5e64e9e2" ], "indie": [ - "zstd_aa@0.1.0+4f16c720", - "tar_aa@0.1.0+9e3fa774", - "act_ghrel@0.1.0+95dbc2b8" + "zstd_aa@0.1.0+993fa832", + "tar_aa@0.1.0+d9cbe4e3", + "act_ghrel@0.1.0+a57a28a0" ], "allowed": {}, "revDepEdges": { - "tar_aa@0.1.0+9e3fa774": [ - "cpy_bs_ghrel@0.1.0+84ecde63" + "tar_aa@0.1.0+d9cbe4e3": [ + "cpy_bs_ghrel@0.1.0+5e64e9e2" ], - "zstd_aa@0.1.0+4f16c720": [ - "cpy_bs_ghrel@0.1.0+84ecde63" + "zstd_aa@0.1.0+993fa832": [ + "cpy_bs_ghrel@0.1.0+5e64e9e2" ], - "cpy_bs_ghrel@0.1.0+84ecde63": [ - "pipi_pypi@0.1.0+076a5b8e" + "cpy_bs_ghrel@0.1.0+5e64e9e2": [ + "pipi_pypi@0.1.0+94a90a13" ] }, "depEdges": { - "cpy_bs_ghrel@0.1.0+84ecde63": [ + "cpy_bs_ghrel@0.1.0+5e64e9e2": [ [ - "tar_aa@0.1.0+9e3fa774", + "tar_aa@0.1.0+d9cbe4e3", "tar_aa" ], [ - "zstd_aa@0.1.0+4f16c720", + "zstd_aa@0.1.0+993fa832", "zstd_aa" ] ], - "pipi_pypi@0.1.0+076a5b8e": [ + "pipi_pypi@0.1.0+94a90a13": [ [ - "cpy_bs_ghrel@0.1.0+84ecde63", + "cpy_bs_ghrel@0.1.0+5e64e9e2", "cpy_bs_ghrel" ] ] @@ -1458,7 +1485,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/act.ts" }, "pipi_pypi@0.1.0": { @@ -1531,7 +1557,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "cpy_bs_ghrel" @@ -1569,7 +1594,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "deps": [ { "name": "tar_aa" @@ -1602,7 +1626,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "tar", "versionExtractFlag": "--version", "versionExtractRegex": "(\\d+\\.\\d+)", @@ -1630,7 +1653,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "execName": "zstd", "versionExtractFlag": "--version", "versionExtractRegex": "v(\\d+\\.\\d+\\.\\d+),", @@ -1669,7 +1691,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/protoc.ts" } } @@ -1685,18 +1706,21 @@ "installGraphs": { "ha": { "all": { - "protoc_ghrel@0.1.0+a7969880": { + "protoc_ghrel@0.1.0+bc42449f": { + "instId": "protoc_ghrel@0.1.0+bc42449f", "portRef": "protoc_ghrel@0.1.0", - "conf": { + "config": { + "version": "v25.1", + "depConfigs": {}, "portRef": "protoc_ghrel@0.1.0" } } }, "user": [ - "protoc_ghrel@0.1.0+a7969880" + "protoc_ghrel@0.1.0+bc42449f" ], "indie": [ - "protoc_ghrel@0.1.0+a7969880" + "protoc_ghrel@0.1.0+bc42449f" ], "allowed": {}, "revDepEdges": {}, @@ -1724,7 +1748,6 @@ ] ], "version": "0.1.0", - "conflictResolution": "deferToNewer", "moduleSpecifier": "file:///data/home/ghjk/ports/protoc.ts" } } diff --git a/ghjk.ts b/ghjk.ts index 32c1e21e..44a9fc58 100644 --- a/ghjk.ts +++ b/ghjk.ts @@ -21,8 +21,8 @@ ghjk.install(); // these are used for developing ghjk ghjk.install( - // act(), - // ...pipi({ packageName: "pre-commit" }), + act(), + ...pipi({ packageName: "pre-commit" }), ); export const secureConfig = ghjk.secureConfig({ diff --git a/modules/ports/mod.ts b/modules/ports/mod.ts index 852ad0db..096dd558 100644 --- a/modules/ports/mod.ts +++ b/modules/ports/mod.ts @@ -1,13 +1,18 @@ export * from "./types.ts"; import { cliffy_cmd } from "../../deps/cli.ts"; -import { $, JSONValue } from "../../utils/mod.ts"; +import { JSONValue } from "../../utils/mod.ts"; +import logger from "../../utils/logger.ts"; import validators from "./types.ts"; import type { PortsModuleConfigX } from "./types.ts"; import type { GhjkCtx, ModuleManifest } from "../types.ts"; import { ModuleBase } from "../mod.ts"; -import { buildInstallGraph, installAndShimEnv, InstallGraph } from "./sync.ts"; -import { installsDbKv } from "./db.ts"; +import { + buildInstallGraph, + installAndShimEnv, + InstallGraph, + syncCtxFromGhjk, +} from "./sync.ts"; export type PortsModuleManifest = { config: PortsModuleConfigX; @@ -16,7 +21,7 @@ export type PortsModuleManifest = { export class PortsModule extends ModuleBase { async processManifest( - _ctx: GhjkCtx, + ctx: GhjkCtx, manifest: ModuleManifest, ) { const res = validators.portsModuleConfig.safeParse(manifest.config); @@ -28,9 +33,11 @@ export class PortsModule extends ModuleBase { }, }); } + await using syncCx = await syncCtxFromGhjk(ctx); + const graph = await buildInstallGraph(syncCx, res.data); return { config: res.data, - graph: await buildInstallGraph(res.data), + graph, }; } command( @@ -47,15 +54,11 @@ export class PortsModule extends ModuleBase { "sync", new cliffy_cmd.Command().description("Syncs the environment.") .action(async () => { - const portsDir = await $.path(ctx.ghjkDir).resolve("ports") - .ensureDir(); - using db = await installsDbKv( - portsDir.resolve("installs.db").toString(), - ); + logger().debug("syncing ports"); + await using syncCx = await syncCtxFromGhjk(ctx); void await installAndShimEnv( - portsDir.toString(), + syncCx, ctx.envDir, - db, manifest.graph, ); }), diff --git a/modules/ports/sync.ts b/modules/ports/sync.ts index 7dc82f23..c5a4d8e9 100644 --- a/modules/ports/sync.ts +++ b/modules/ports/sync.ts @@ -7,9 +7,10 @@ import type { DepArts, DownloadArtifacts, InstallArtifacts, - InstallConfigLite, InstallConfigLiteX, + InstallConfigResolvedX, PortArgsBase, + PortDep, PortManifestX, PortsModuleConfigX, } from "./types.ts"; @@ -21,24 +22,55 @@ import { DePromisify, getInstallHash, getPortRef, + sameFsTmpRoot, } from "../../utils/mod.ts"; -import type { InstallsDb } from "./db.ts"; +import { type InstallsDb, installsDbKv } from "./db.ts"; +import type { GhjkCtx } from "../types.ts"; const logger = getLogger(import.meta); +type SyncCtx = { + installsPath: string; + downloadsPath: string; + tmpPath: string; + db: InstallsDb; +}; + +export async function syncCtxFromGhjk(ctx: GhjkCtx) { + const portsPath = await $.path(ctx.ghjkDir).resolve("ports") + .ensureDir(); + // ensure the req + const [installsPath, downloadsPath, tmpPath] = ( + await Promise.all([ + portsPath.join("installs").ensureDir(), + portsPath.join("downloads").ensureDir(), + sameFsTmpRoot(portsPath.toString()), + ]) + ).map($.pathToString); + const db = await installsDbKv( + portsPath.resolve("installs.db").toString(), + ); + return { + db, + installsPath, + downloadsPath, + tmpPath, + async [Symbol.asyncDispose]() { + db[Symbol.dispose](); + await $.removeIfExists(tmpPath); + }, + }; +} /* */ export async function installAndShimEnv( - portsDir: string, + cx: SyncCtx, envDir: string, - installsDb: InstallsDb, graph: InstallGraph, createShellLoaders = true, ) { - logger.debug("syncing ports"); const installArts = await installAll( - portsDir, - installsDb, + cx, graph, ); // create the shims for the user's environment @@ -135,8 +167,7 @@ export async function installAndShimEnv( } export async function installAll( - portsDir: string, - installsDb: InstallsDb, + cx: SyncCtx, graph: InstallGraph, ) { const installCtx = { @@ -165,7 +196,7 @@ export async function installAll( const depArts = installCtx.artifacts.get(depInstallId); if (!depArts) { throw new Error( - `artifacts not found for plug dep "${depInstallId}" when installing "${installId}"`, + `artifacts not found for port dep "${depInstallId}" when installing "${installId}"`, ); } const depShimDir = $.path(depShimsRootPath).resolve(depInstallId); @@ -222,22 +253,12 @@ export async function installAll( return readyParents; }, }; - logger.debug("syncing ports"); - const portsPath = $.path(portsDir); - // ensure the req - const [installsPath, downloadsPath, tmpPath] = ( - await Promise.all([ - portsPath.join("installs").ensureDir(), - portsPath.join("downloads").ensureDir(), - movebleTmpRoot(portsDir), - ]) - ).map($.pathToString); // start from the ports with no build deps const pendingInstalls = [...graph.indie]; while (pendingInstalls.length > 0) { const installId = pendingInstalls.pop()!; - const cached = await installsDb.get(installId); + const cached = await cx.db.get(installId); let thisArtifacts; // we skip it if it's already installed @@ -253,23 +274,23 @@ export async function installAll( // shims for their exports const { totalDepArts, depShimsRootPath } = await installCtx .readyDepArts( - tmpPath, + cx.tmpPath, installId, ); const stageArgs = { installId, - installPath: std_path.resolve(installsPath, installId), - downloadPath: std_path.resolve(downloadsPath, installId), - tmpPath, - conf: inst.conf, + installPath: std_path.resolve(cx.installsPath, installId), + downloadPath: std_path.resolve(cx.downloadsPath, installId), + tmpPath: cx.tmpPath, + config: inst.config, manifest, depArts: totalDepArts, }; const dbRow = { installId, - conf: inst.conf, + conf: inst.config, manifest, }; let downloadArts; @@ -285,7 +306,7 @@ export async function installAll( } catch (err) { throw new Error(`error downloading ${installId}`, { cause: err }); } - await installsDb.set(installId, { + await cx.db.set(installId, { ...dbRow, progress: "downloaded", downloadArts, @@ -302,7 +323,7 @@ export async function installAll( } catch (err) { throw new Error(`error installing ${installId}`, { cause: err }); } - await installsDb.set(installId, { + await cx.db.set(installId, { ...dbRow, progress: "installed", downloadArts, @@ -313,49 +334,21 @@ export async function installAll( installCtx.artifacts.set(installId, thisArtifacts); pendingInstalls.push(...installCtx.installDone(installId)); } - await $.removeIfExists(tmpPath); return installCtx.artifacts; } -/* * - * This returns a tmp path that's guaranteed to be - * on the same file system as targetDir by - * checking if $TMPDIR satisfies that constraint - * or just pointing to targetDir/tmp - * This is handy for making moves atomics from - * tmp dirs to to locations within targetDir - * - * Make sure to remove the dir after use - */ -async function movebleTmpRoot(targetDir: string, targetTmpDirName = "dir") { - const defaultTmp = Deno.env.get("TMPDIR"); - const targetPath = $.path(targetDir); - if (!defaultTmp) { - // this doens't return a unique tmp dir on every sync - // this allows subsequent syncs to clean up after - // some previously failing sync as this is not a system managed - // tmp dir but this means two concurrent syncing will clash - // TODO: mutex file to prevent block concurrent syncinc - return await targetPath.join(targetTmpDirName).ensureDir(); - } - const defaultTmpPath = $.path(defaultTmp); - if ((await targetPath.stat())?.dev != (await defaultTmpPath.stat())?.dev) { - return await targetPath.join(targetTmpDirName).ensureDir(); - } - // when using the system managed tmp dir, we create a new tmp dir in it - // we don't care if the sync fails before it cleans as the system will - // take care of it - return $.path(await Deno.makeTempDir({ prefix: "ghjk_sync" })); -} - export type InstallGraph = DePromisify>; // this returns a data structure containing all the info // required for installation including the dependency graph -export async function buildInstallGraph(cx: PortsModuleConfigX) { +export async function buildInstallGraph( + cx: SyncCtx, + portsConfig: PortsModuleConfigX, +) { type GraphInstConf = { + instId: string; portRef: string; - conf: InstallConfigLite; + config: InstallConfigResolvedX; }; // this is all referring to port dependencies // TODO: runtime dependencies @@ -400,20 +393,20 @@ export async function buildInstallGraph(cx: PortsModuleConfigX) { const foundInstalls: GraphInstConf[] = []; // collect the user specified insts first - for (const inst of cx.installs) { + for (const inst of portsConfig.installs) { const { port: manifest, ...instLiteBase } = inst; const portRef = addPort(manifest); const instLite = validators.installConfigLite.parse({ ...instLiteBase, portRef: getPortRef(manifest), }); - // const port = getPortImpl(manifest); - // const listAllArgs = {}; - // if (instLite.version) { - // port.listAll({}) - // } else { - // } - const instId = await getInstallHash(instLite); + const resolvedConfig = await resolveConfig( + cx, + portsConfig, + manifest, + instLite, + ); + const instId = await getInstallHash(resolvedConfig); // no dupes allowed in user specified insts if (graph.user.includes(instId)) { @@ -424,7 +417,7 @@ export async function buildInstallGraph(cx: PortsModuleConfigX) { ); } graph.user.push(instId); - foundInstalls.push({ portRef, conf: instLite }); + foundInstalls.push({ instId, portRef, config: resolvedConfig }); } // process each port's dependency trees @@ -441,10 +434,9 @@ export async function buildInstallGraph(cx: PortsModuleConfigX) { ); } - const installId = await getInstallHash(inst.conf); - + const installId = inst.instId; // there might be multiple instances of an install at this point - // due to a single plugin being a dependency to multiple others + // due to a single install being a dependency to multiple others const conflict = graph.all[installId]; if (conflict) { continue; @@ -458,29 +450,20 @@ export async function buildInstallGraph(cx: PortsModuleConfigX) { // this goes into graph.depEdges const deps: [string, string][] = []; for (const depId of manifest.deps) { - const { manifest: depPort, defaultInst: defaultDepInstall } = - cx.allowedDeps[depId.name]; + const { manifest: depPort } = portsConfig.allowedDeps[depId.name]; if (!depPort) { throw new Error( - `unrecognized dependency "${depId.name}" specified by plug "${manifest.name}@${manifest.version}"`, + `unrecognized dependency "${depId.name}" specified by port "${manifest.name}@${manifest.version}"`, ); } + const portRef = addPort(depPort); // get the install config of dependency - let depInstall; - { - // install configuration of allowed dep ports - // can be overriden by dependent ports - const res = validators.installConfigLite.safeParse( - inst.conf.depConfigs?.[depId.name] ?? defaultDepInstall, - ); - if (!res.success) { - throw new Error( - `error parsing depConfig for "${depId.name}" as specified by "${installId}": ${res.error}`, - ); - } - depInstall = res.data; - } + // the conf is of the resolved kind which means + // it's deps are also resolved + const depInstall = validators.installConfigResolved.parse( + inst.config.depConfigs![depId.name], + ); const depInstallId = await getInstallHash(depInstall); // check for cycles @@ -496,8 +479,11 @@ export async function buildInstallGraph(cx: PortsModuleConfigX) { // only add the install configuration for this dep port // if specific hash hasn't seen before if (!graph.all[depInstallId]) { - const portRef = addPort(depPort); - foundInstalls.push({ conf: depInstall, portRef }); + foundInstalls.push({ + config: depInstall, + portRef, + instId: depInstallId, + }); } deps.push([depInstallId, depPort.name]); @@ -514,6 +500,293 @@ export async function buildInstallGraph(cx: PortsModuleConfigX) { return graph; } +// This takes user specified InstallConfigs and resolves +// their versions to a known, installable version +// It also resolves any dependencies that the config specifies +async function resolveConfig( + cx: SyncCtx, + portsConfig: PortsModuleConfigX, + manifest: PortManifestX, + config: InstallConfigLiteX, +) { + // resolve and install the resolutionDeps first so that we + // can invoke listAll and latestStable + const resolvedResolutionDeps = [] as [string, string][]; + for (const dep of manifest.resolutionDeps ?? []) { + const { manifest: depMan, config: depConf } = getDepConfig( + portsConfig, + manifest, + config, + dep, + true, + ); + + // get the version resolved config of the dependency + const depInstId = await resolveAndInstall( + cx, + portsConfig, + depMan, + depConf, + ); + resolvedResolutionDeps.push([depInstId.installId, depMan.name]); + } + + const depShimsRootPath = await Deno.makeTempDir({ + dir: cx.tmpPath, + prefix: `shims_resDeps_${manifest.name}_`, + }); + const resolutionDepArts = await getShimmedDepArts( + cx, + depShimsRootPath, + resolvedResolutionDeps, + ); + + // finally resolve the versino + let version; + // TODO: fuzzy matching + const port = getPortImpl(manifest); + const listAllArgs = { + depArts: resolutionDepArts, + config, + manifest, + }; + if (config.version) { + const allVersions = await port.listAll(listAllArgs); + if (!allVersions.includes(config.version)) { + throw new Error(`error resolving verison: not found`, { + cause: { config, manifest }, + }); + } + version = config.version; + } else { + const latestStable = await port.latestStable(listAllArgs); + version = latestStable; + } + await $.removeIfExists(depShimsRootPath); + + // now we resolve the remaning deps + // TODO: port version dependent portDep resolution + // e.g. use python-2.7 if foo is resolved to <1.0 or use + // python-3.x if foo is resolved to >1.0 + const resolveDepConfigs = {} as Record; + for (const dep of manifest.deps ?? []) { + const { manifest: depMan, config: depConf } = getDepConfig( + portsConfig, + manifest, + config, + dep, + ); + // get the version resolved config of the dependency + const depInstall = await resolveConfig( + cx, + portsConfig, + depMan, + depConf, + ); + resolveDepConfigs[dep.name] = depInstall; + } + + return validators.installConfigResolved.parse({ + ...config, + depConfigs: resolveDepConfigs, + version, + }); +} + +// This gets either the dependency InstallConfig as specified by +// config.depPorts[depId] or the default InstallConfig specified +// for the portsConfig.allowedDeps +// No version resolution takes place +function getDepConfig( + portsConfig: PortsModuleConfigX, + manifest: PortManifestX, + config: InstallConfigLiteX, + depId: PortDep, + resolutionDep = false, +) { + const { manifest: depPort, defaultInst: defaultDepInstall } = + portsConfig.allowedDeps[depId.name]; + if (!depPort) { + throw new Error( + `unrecognized dependency "${depId.name}" specified by port "${manifest.name}@${manifest.version}"`, + ); + } + // install configuration of an allowed dep port + // can be overriden by dependent ports + const res = validators.installConfigLite.safeParse( + (resolutionDep ? config.resolutionDepConfigs : config.depConfigs) + ?.[depId.name] ?? defaultDepInstall, + ); + if (!res.success) { + throw new Error( + `error parsing depConfig for "${depId.name}" as specified by install of "${manifest.name}": ${res.error}`, + { + cause: { + config, + manifest, + zodErr: res.error, + }, + }, + ); + } + return { config: res.data, manifest: depPort }; +} + +/// This is a simpler version of the graph +/// based installer that the rest of this module implements +/// it resolves and installs a single config (and it's deps). +/// This primarily is used to install the manifest.resolutionDeps +/// which are required to do version resolution when building the +/// graph +/// FIXME: the usage of this function implies that resolution +/// will be redone if a config specfied by different resolutionDeps +/// Consider introducing a memoization scheme +async function resolveAndInstall( + cx: SyncCtx, + portsConfig: PortsModuleConfigX, + manifest: PortManifestX, + configLite: InstallConfigLiteX, +) { + const config = await resolveConfig(cx, portsConfig, manifest, configLite); + const installId = await getInstallHash(config); + + const cached = await cx.db.get(installId); + // we skip it if it's already installed + if (cached && cached.progress == "installed") { + logger.debug("already installed, skipping", installId); + } else { + const depShimsRootPath = await Deno.makeTempDir({ + dir: cx.tmpPath, + prefix: `shims_${installId}`, + }); + // readies all the exports of the port's deps including + // shims for their exports + const totalDepArts = await getShimmedDepArts( + cx, + depShimsRootPath, + await Promise.all( + manifest.deps?.map( + async (dep) => { + const depConfig = getDepConfig(portsConfig, manifest, config, dep); + // we not only resolve but install the dep here + const { installId } = await resolveAndInstall( + cx, + portsConfig, + depConfig.manifest, + depConfig.config, + ); + return [installId, dep.name]; + }, + ) ?? [], + ), + ); + + const stageArgs = { + installId, + installPath: std_path.resolve(cx.installsPath, installId), + downloadPath: std_path.resolve(cx.installsPath, installId), + tmpPath: cx.tmpPath, + config: config, + manifest, + depArts: totalDepArts, + }; + + const dbRow = { + installId, + conf: config, + manifest, + }; + let downloadArts; + if (cached) { + logger.debug("already downloaded, skipping to install", installId); + // download step must have completed if there's a cache hit + downloadArts = cached.downloadArts; + } else { + try { + downloadArts = await doDownloadStage({ + ...stageArgs, + }); + } catch (err) { + throw new Error(`error downloading ${installId}`, { cause: err }); + } + await cx.db.set(installId, { + ...dbRow, + progress: "downloaded", + downloadArts, + }); + } + + let installArtifacts; + try { + installArtifacts = await doInstallStage( + { + ...stageArgs, + ...downloadArts, + }, + ); + } catch (err) { + throw new Error(`error installing ${installId}`, { cause: err }); + } + await cx.db.set(installId, { + ...dbRow, + progress: "installed", + downloadArts, + installArts: installArtifacts, + }); + await $.removeIfExists(depShimsRootPath); + } + return { installId, config }; +} + +// This assumes that the installs are already in the db +async function getShimmedDepArts( + cx: SyncCtx, + shimsRootPath: string, + installs: [string, string][], +) { + const totalDepArts: DepArts = {}; + for ( + const [installId, portName] of installs + ) { + const installRow = await cx.db.get(installId); + if (!installRow || !installRow.installArts) { + throw new Error( + `artifacts not found for "${installId}" not found in db when shimming totalDepArts`, + { + cause: { installs }, + }, + ); + } + const installArts = installRow.installArts; + const shimDir = $.path(shimsRootPath).resolve(installId); + const [binShimDir, libShimDir, includeShimDir] = (await Promise.all([ + shimDir.join("bin").ensureDir(), + shimDir.join("lib").ensureDir(), + shimDir.join("include").ensureDir(), + ])).map($.pathToString); + + totalDepArts[portName] = { + execs: await shimLinkPaths( + installArts.binPaths, + installArts.installPath, + binShimDir, + ), + libs: await shimLinkPaths( + installArts.libPaths, + installArts.installPath, + libShimDir, + ), + includes: await shimLinkPaths( + installArts.includePaths, + installArts.installPath, + includeShimDir, + ), + env: installArts.env, + }; + } + return totalDepArts; +} + /// This expands globs found in the targetPaths async function shimLinkPaths( targetPaths: string[], @@ -570,50 +843,45 @@ export function getPortImpl(manifest: PortManifestX) { ); } else { throw new Error( - `unsupported plugin type "${(manifest as unknown as any).ty}": ${ + `unsupported port type "${(manifest as unknown as any).ty}": ${ $.inspect(manifest) }`, ); } } +type DownloadStageArgs = { + installId: string; + installPath: string; + downloadPath: string; + tmpPath: string; + config: InstallConfigResolvedX; + manifest: PortManifestX; + depArts: DepArts; +}; + async function doDownloadStage( { installId, installPath, downloadPath, tmpPath, - conf, + config, manifest, depArts, - }: { - installId: string; - installPath: string; - downloadPath: string; - tmpPath: string; - conf: InstallConfigLiteX; - manifest: PortManifestX; - depArts: DepArts; - }, + }: DownloadStageArgs, ) { logger.debug("downloading", { installId, installPath, downloadPath, - conf, + config, port: manifest, }); const port = getPortImpl(manifest); - const installVersion = validators.string.parse( - conf.version ?? - await port.latestStable({ - depArts, - manifest, - config: conf, - }), - ); + const installVersion = config.version; logger.info(`downloading ${installId}:${installVersion}`); const tmpDirPath = await Deno.makeTempDir({ @@ -625,7 +893,7 @@ async function doDownloadStage( installVersion: installVersion, depArts, platform: Deno.build, - config: conf, + config: config, manifest, downloadPath, tmpDirPath, @@ -639,47 +907,38 @@ async function doDownloadStage( return out; } +type InstallStageArgs = DownloadStageArgs; + async function doInstallStage( { installId, installPath, downloadPath, tmpPath, - conf, + config, manifest, depArts, - installVersion, - }: { - installId: string; - installPath: string; - downloadPath: string; - tmpPath: string; - conf: InstallConfigLite; - manifest: PortManifestX; - depArts: DepArts; - installVersion: string; - }, + }: InstallStageArgs, ) { logger.debug("installing", { installId, installPath, downloadPath, - conf, + config, port: manifest, }); const port = getPortImpl(manifest); + const installVersion = config.version; const baseArgs: PortArgsBase = { installPath, installVersion, depArts, platform: Deno.build, - config: conf, + config: config, manifest, }; - logger.debug("baseArgs", installId, baseArgs); - { logger.info(`installing ${installId}:${installVersion}`); const tmpDirPath = await Deno.makeTempDir({ diff --git a/modules/ports/types.ts b/modules/ports/types.ts index b71bf546..224f099f 100644 --- a/modules/ports/types.ts +++ b/modules/ports/types.ts @@ -14,6 +14,11 @@ const portDep = zod.object({ name: portName, }); +const portDepFat = portDep.merge(zod.object({ + // FIXME: figure out cyclically putting `installConfigLite` here + config: zod.unknown(), +})); + export const ALL_OS = [ "linux", "darwin", @@ -40,12 +45,13 @@ const portManifestBase = zod.object({ .refine((str) => semver.parse(str), { message: "invalid semver string", }), - conflictResolution: zod - .enum(["deferToNewer", "override"]) - .nullish() - // default value set after transformation - .default("deferToNewer"), + // conflictResolution: zod + // .enum(["deferToNewer", "override"]) + // .nullish() + // // default value set after transformation + // .default("deferToNewer"), deps: zod.array(portDep).nullish(), + resolutionDeps: zod.array(portDep).nullish(), }).passthrough(); const denoWorkerPortManifest = portManifestBase.merge( @@ -97,6 +103,10 @@ const installConfigBase = installConfigSimple.merge(zod.object({ // FIXME: figure out cyclically putting `installConfigLite` here zod.unknown(), ).nullish(), + resolutionDepConfigs: zod.record( + portName, + zod.unknown(), + ).nullish(), })).passthrough(); const installConfigBaseFat = installConfigBase.merge(zod.object({ @@ -118,6 +128,16 @@ const installConfigLite = // ]); const installConfigFat = stdInstallConfigFat; +const installConfigResolved = installConfigLite.merge(zod.object({ + // NOTE: version is no longer nullish + version: zod.string(), + // depConfigs: zod.record( + // portName, + // // FIXME: figure out cyclically putting `installConfigResolved` here + // zod.object({ version: zod.string() }).passthrough(), + // ), +})).passthrough(); + // NOTE: zod unions are tricky. It'll parse with the first schema // in the array that parses. And if this early schema is a subset // of its siblings (and it doesn't have `passthrough`), it will discard @@ -159,6 +179,7 @@ const validators = { archEnum, portName, portDep, + portDepFat, portManifestBase, denoWorkerPortManifest, ambientAccessPortManifest, @@ -171,6 +192,7 @@ const validators = { installConfigFat, installConfigLite, installConfig, + installConfigResolved, portManifest, portsModuleConfigBase, portsModuleSecureConfig, @@ -213,6 +235,7 @@ export type PortManifestX = zod.infer< /// PortDeps are used during the port build/install process export type PortDep = zod.infer; +export type PortDepFat = zod.infer; export type InstallConfigSimple = zod.input< typeof validators.installConfigSimple @@ -235,6 +258,12 @@ export type InstallConfigLiteX = zod.infer; export type InstallConfig = zod.input; // Describes a single installation done by a specific plugin. export type InstallConfigX = zod.infer; +export type InstallConfigResolved = zod.input< + typeof validators.installConfigResolved +>; +export type InstallConfigResolvedX = zod.infer< + typeof validators.installConfigResolved +>; export type PortsModuleConfigBase = zod.infer< typeof validators.portsModuleConfigBase diff --git a/modules/tasks/mod.ts b/modules/tasks/mod.ts index d387c082..a2d1256c 100644 --- a/modules/tasks/mod.ts +++ b/modules/tasks/mod.ts @@ -14,8 +14,8 @@ import { buildInstallGraph, installAndShimEnv, InstallGraph, + syncCtxFromGhjk, } from "../ports/sync.ts"; -import { installsDbKv } from "../ports/db.ts"; export type TaskModuleManifest = { config: TasksModuleConfigX; @@ -23,7 +23,7 @@ export type TaskModuleManifest = { }; export class TasksModule extends ModuleBase { async processManifest( - _ctx: GhjkCtx, + ctx: GhjkCtx, manifest: ModuleManifest, ) { const res = validators.tasksModuleConfig.safeParse(manifest.config); @@ -35,6 +35,7 @@ export class TasksModule extends ModuleBase { }, }); } + await using syncCx = await syncCtxFromGhjk(ctx); return { config: res.data, installGraphs: Object.fromEntries( @@ -42,10 +43,13 @@ export class TasksModule extends ModuleBase { Object.entries(res.data.tasks) .map(async ([name, task]) => [ name, - await buildInstallGraph({ - installs: task.env.installs, - allowedDeps: task.env.allowedPortDeps, - }), + await buildInstallGraph( + syncCx, + { + installs: task.env.installs, + allowedDeps: task.env.allowedPortDeps, + }, + ), ]), ), ), @@ -119,18 +123,13 @@ export async function execTask( taskEnv: TaskEnvX, installGraph: InstallGraph, ): Promise { - const portsDir = await $.path(ctx.ghjkDir).resolve("ports") - .ensureDir(); - using db = await installsDbKv( - portsDir.resolve("installs.db").toString(), - ); + await using syncCx = await syncCtxFromGhjk(ctx); const taskEnvDir = await Deno.makeTempDir({ prefix: `ghjkTaskEnv_${name}_`, }); const { env: installEnvs } = await installAndShimEnv( - portsDir.toString(), + syncCx, taskEnvDir, - db, installGraph, ); logger().info("executing", name); diff --git a/ports/asdf.ts b/ports/asdf.ts index 2ce0012a..d2427db1 100644 --- a/ports/asdf.ts +++ b/ports/asdf.ts @@ -23,6 +23,12 @@ export const manifest = { version: "0.1.0", moduleSpecifier: import.meta.url, deps: [std_ports.curl_aa, std_ports.git_aa, std_ports.asdf_plugin_git], + // NOTE: we require the same port set for version resolution as well + resolutionDeps: [ + std_ports.curl_aa, + std_ports.git_aa, + std_ports.asdf_plugin_git, + ], platforms: osXarch(["linux", "darwin"], ["x86_64", "aarch64"]), }; @@ -43,15 +49,17 @@ export default function conf( const { port: pluginPort, ...liteConf } = asdf_plugin_git({ pluginRepo: config.pluginRepo, }); + const depConfigs = { + [std_ports.asdf_plugin_git.name]: { + ...liteConf, + portRef: getPortRef(pluginPort), + }, + }; return { ...confValidator.parse(config), port: manifest, - depConfigs: { - [std_ports.asdf_plugin_git.name]: { - ...liteConf, - portRef: getPortRef(pluginPort), - }, - }, + depConfigs, + resolutionDepConfigs: depConfigs, }; } diff --git a/ports/asdf_plugin_git.ts b/ports/asdf_plugin_git.ts index 84d4cb60..3e74aac5 100644 --- a/ports/asdf_plugin_git.ts +++ b/ports/asdf_plugin_git.ts @@ -22,6 +22,7 @@ export const manifest = { version: "0.1.0", moduleSpecifier: import.meta.url, deps: [git_aa_id], + resolutionDeps: [git_aa_id], platforms: osXarch(["linux", "darwin", "windows"], ["aarch64", "x86_64"]), }; diff --git a/utils/mod.ts b/utils/mod.ts index 73ff95bb..2f32ce23 100644 --- a/utils/mod.ts +++ b/utils/mod.ts @@ -4,7 +4,7 @@ import logger, { isColorfulTty } from "./logger.ts"; import type { DepArts, DownloadArgs, - InstallConfigLite, + InstallConfigResolvedX, OsEnum, PortDep, PortManifest, @@ -22,7 +22,7 @@ export type JSONObject = { [key: string]: JSONValue }; export type JSONArray = JSONValue[]; export function dbg(val: T, ...more: unknown[]) { - logger().debug(() => val, ...more); + logger().debug("DBG", val, ...more); return val; } @@ -124,7 +124,7 @@ export function getPortRef(manifest: PortManifest) { return `${manifest.name}@${manifest.version}`; } -export async function getInstallHash(install: InstallConfigLite) { +export async function getInstallHash(install: InstallConfigResolvedX) { const hashBuf = await jsonHash.digest("SHA-256", install as jsonHash.Tree); const hashHex = bufferToHex(hashBuf).slice(0, 8); return `${install.portRef}+${hashHex}`; @@ -333,3 +333,37 @@ export async function downloadFile( await tmpFilePath.copyFile(fileDwnPath); } + +/* * + * This returns a tmp path that's guaranteed to be + * on the same file system as targetDir by + * checking if $TMPDIR satisfies that constraint + * or just pointing to targetDir/tmp + * This is handy for making moves atomics from + * tmp dirs to to locations within targetDir + * + * Make sure to remove the dir after use + */ +export async function sameFsTmpRoot( + targetDir: string, + targetTmpDirName = "tmp", +) { + const defaultTmp = Deno.env.get("TMPDIR"); + const targetPath = $.path(targetDir); + if (!defaultTmp) { + // this doens't return a unique tmp dir on every sync + // this allows subsequent syncs to clean up after + // some previously failing sync as this is not a system managed + // tmp dir but this means two concurrent syncing will clash + // TODO: mutex file to prevent block concurrent syncinc + return await targetPath.join(targetTmpDirName).ensureDir(); + } + const defaultTmpPath = $.path(defaultTmp); + if ((await targetPath.stat())?.dev != (await defaultTmpPath.stat())?.dev) { + return await targetPath.join(targetTmpDirName).ensureDir(); + } + // when using the system managed tmp dir, we create a new tmp dir in it + // we don't care if the sync fails before it cleans as the system will + // take care of it + return $.path(await Deno.makeTempDir({ prefix: "ghjk_sync" })); +}