-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: use tar-fs to produce a tar stream
Rather than building with CLI GNU tar, because that way we can control the contents more closely
- Loading branch information
Dominic Scheirlinck
committed
Jan 14, 2022
1 parent
21f5222
commit c4c9085
Showing
9 changed files
with
188 additions
and
95 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,72 +1,108 @@ | ||
| import fs from 'fs'; | ||
| import stream, { pipeline as pipelineSync } from 'stream'; | ||
| import { promisify } from 'util'; | ||
| import { promises as fs, PathLike } from 'fs'; | ||
| import path from 'path'; | ||
| import debug from 'debug'; | ||
| import _ from 'lodash'; | ||
| import split from 'split2'; | ||
| import { globSet } from '../util/glob'; | ||
| import { depthSort } from '../util/tar'; | ||
| import { count } from '../util/helper'; | ||
|
|
||
| const pipeline = promisify(pipelineSync); | ||
| const exists = async (file: string) => { | ||
| try { | ||
| await fs.stat(file); | ||
| return true; | ||
| } catch (err) { | ||
| return false; | ||
| } | ||
| }; | ||
|
|
||
| const log = debug('monofo:artifact:matcher'); | ||
|
|
||
| /** | ||
| * Utility class that can act as a writable stream of file name chunks, or receive globs | ||
| */ | ||
| class MatchedFiles extends stream.Writable { | ||
| constructor(private _matched: string[] = []) { | ||
| super({ | ||
| objectMode: true, | ||
| write: (chunk: string, _encoding, next) => { | ||
| this._matched.push(chunk); | ||
| next(); | ||
| }, | ||
| }); | ||
| } | ||
| export interface PathsToPack { | ||
| [path: string]: { recurse: boolean }; | ||
| } | ||
|
|
||
| async addGlobs(globs: string[]): Promise<void> { | ||
| this._matched = [...this._matched, ...(await globSet(_.castArray(globs), { matchBase: false }))]; | ||
| } | ||
| function isRoot(p: string): boolean { | ||
| return p === '' || p === '/' || p === '.'; | ||
| } | ||
|
|
||
| export function addIntermediateDirectories(toPack: PathsToPack): PathsToPack { | ||
| const repacked: PathsToPack = {}; | ||
|
|
||
| const addWithParents = (p: string, recurse: boolean): void => { | ||
| if (isRoot(p)) { | ||
| return; | ||
| } | ||
|
|
||
| const parent = path.dirname(p); | ||
|
|
||
| get matched() { | ||
| return this._matched; | ||
| if (!isRoot(parent) && !(parent in repacked)) { | ||
| log(`Adding intermediate directory to included paths to upload: ${parent}`); | ||
| addWithParents(parent, false); | ||
| } | ||
|
|
||
| repacked[p] = { recurse }; | ||
| }; | ||
|
|
||
| for (const [p, { recurse }] of Object.entries(toPack)) { | ||
| addWithParents(p, recurse); | ||
| } | ||
|
|
||
| return repacked; | ||
| } | ||
|
|
||
| export async function filesToUpload({ | ||
| /** | ||
| * The files will be passed to tar in the order shown, and then tar will | ||
| * recurse into each entry if it's a directory (because --recursive is the | ||
| * default) - it should use the --sort argument (if your tar is new enough) | ||
| * to sort the eventual input file list, but they'll still be ordered according | ||
| * to the order of this files argument | ||
| * | ||
| * This is problematic if the paths don't contain every intermediate directory | ||
| * | ||
| * To fix this, and make the eventual tar compatible with catar, we do the | ||
| * recursion into files ourselves. | ||
| */ | ||
| export async function pathsToUpload({ | ||
| filesFrom, | ||
| globs, | ||
| useNull = false, | ||
| }: { | ||
| filesFrom?: string; | ||
| globs?: string[]; | ||
| useNull?: boolean; | ||
| }): Promise<string[]> { | ||
| }): Promise<Record<string, { recurse: boolean }>> { | ||
| if (!filesFrom && !globs) { | ||
| return []; | ||
| return {}; | ||
| } | ||
|
|
||
| const matched = new MatchedFiles(); | ||
| const matching: Promise<void>[] = []; | ||
|
|
||
| if (globs) { | ||
| matching.push(matched.addGlobs(globs)); | ||
| } | ||
| const paths: Record<string, { recurse: boolean }> = Object.fromEntries( | ||
| ( | ||
| await globSet( | ||
| _.castArray(globs).filter((v) => v), | ||
| { matchBase: false } | ||
| ) | ||
| ).map((p) => [`./${p}`, { recurse: false }]) | ||
| ); | ||
|
|
||
| if (filesFrom) { | ||
| if (filesFrom !== '-' && !fs.existsSync(filesFrom)) { | ||
| if (filesFrom !== '-' && !(await exists(filesFrom))) { | ||
| throw new Error(`Could not find file to read file list from: ${filesFrom}`); | ||
| } | ||
|
|
||
| log(`Reading from ${filesFrom !== '-' ? filesFrom : 'stdin'}`); | ||
| const source: stream.Readable = | ||
| filesFrom === '-' ? process.stdin : fs.createReadStream(filesFrom, { encoding: 'utf8', autoClose: true }); | ||
| const source: string = | ||
| filesFrom === '-' | ||
| ? await fs.readFile(0 as unknown as PathLike, 'utf8') // 0 is process.stdin | ||
| : await fs.readFile(filesFrom, { encoding: 'utf8' }); | ||
|
|
||
| for (const p of source.split(useNull ? '\x00' : '\n').filter((v) => v)) { | ||
| paths[p] = { recurse: true }; | ||
| } | ||
| } | ||
|
|
||
| matching.push(pipeline(source, split(useNull ? '\x00' : '\n'), matched)); | ||
| // TODO: pipeline is sync here anyway, so this can be simplified | ||
| if (!Object.keys(paths).every((p) => p.startsWith('./'))) { | ||
| throw new Error('Expected to be given only relative paths to recurse, relative to CWD'); | ||
| } | ||
|
|
||
| await Promise.all(matching); | ||
| return depthSort(matched.matched); | ||
| log(`Globs and file input matched ${count(Object.keys(paths), 'path')}`); | ||
| return addIntermediateDirectories(paths); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.