Skip to content

Commit 09b922a

Browse files
Ian2020dbrgnjwillikers
committed
node: Add support for git sources
Co-authored-by: Danilo Bargen <[email protected]> Co-authored-by: Jordan Williams <[email protected]>
1 parent c93d5b8 commit 09b922a

File tree

11 files changed

+784
-78
lines changed

11 files changed

+784
-78
lines changed

node/flatpak_node_generator/providers/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
'git': {},
1717
'git+http': {'scheme': 'http'},
1818
'git+https': {'scheme': 'https'},
19+
'git+ssh': {'scheme': 'https'},
1920
}
2021

2122

@@ -33,6 +34,14 @@ def parse_git_source(self, version: str, from_: Optional[str] = None) -> GitSour
3334
if not new_url.netloc:
3435
path = new_url.path.split('/')
3536
new_url = new_url._replace(netloc=path[0], path='/'.join(path[1:]))
37+
# Replace https://[email protected]:ianstormtaylor/to-camel-case.git
38+
# with https://[email protected]/ianstormtaylor/to-camel-case.git
39+
# for git+ssh URLs
40+
if ':' in new_url.netloc:
41+
netloc_split = new_url.netloc.split(':')
42+
new_url = new_url._replace(
43+
netloc=netloc_split[0], path=f'/{netloc_split[1]}{new_url.path}'
44+
)
3645

3746
return GitSource(
3847
original=original_url.geturl(),

node/flatpak_node_generator/providers/npm.py

Lines changed: 112 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
NamedTuple,
99
Optional,
1010
Set,
11+
Tuple,
1112
Type,
1213
)
1314

@@ -79,11 +80,17 @@ def _process_packages_v1(
7980
elif version_url.scheme == 'file':
8081
source = LocalSource(path=version_url.path)
8182
else:
82-
integrity = Integrity.parse(info['integrity'])
83+
integrity = None
84+
if 'integrity' in info:
85+
integrity = Integrity.parse(info['integrity'])
86+
8387
if 'resolved' in info:
84-
source = ResolvedSource(
85-
resolved=info['resolved'], integrity=integrity
86-
)
88+
if info['resolved'].startswith('git+'):
89+
source = self.parse_git_source(info['resolved'])
90+
else:
91+
source = ResolvedSource(
92+
resolved=info['resolved'], integrity=integrity
93+
)
8794
elif version_url.scheme in {'http', 'https'}:
8895
source = PackageURLSource(resolved=version, integrity=integrity)
8996
else:
@@ -104,20 +111,15 @@ def _process_packages_v2(
104111
# NOTE We're not interested in symlinks, NPM will create them at install time
105112
# but we still could collect package symlinks anyway just for completeness
106113
continue
114+
if install_path == '':
115+
# The root project is typically listed with a key of ''
116+
continue
107117

108118
name = info.get('name')
109119

110120
source: PackageSource
111-
package_json_path = lockfile.parent / install_path / 'package.json'
112-
if (
113-
'node_modules' not in install_path.split('/')
114-
and package_json_path.exists()
115-
):
116-
source = LocalSource(path=install_path)
117-
if name is None:
118-
with package_json_path.open('rb') as fp:
119-
name = json.load(fp)['name']
120-
elif 'resolved' in info:
121+
122+
if 'resolved' in info:
121123
resolved_url = urllib.parse.urlparse(info['resolved'])
122124
if resolved_url.scheme == 'file':
123125
source = LocalSource(path=resolved_url.path)
@@ -130,14 +132,10 @@ def _process_packages_v2(
130132
integrity=integrity, resolved=info['resolved']
131133
)
132134
elif resolved_url.scheme.startswith('git+'):
133-
raise NotImplementedError(
134-
'Git sources in lockfile v2 format are not supported yet'
135-
f' (package {install_path} in {lockfile})'
136-
)
135+
source = self.parse_git_source(info['resolved'])
137136
else:
138-
raise NotImplementedError(
139-
f"Don't know how to handle package {install_path} in {lockfile}"
140-
)
137+
source = LocalSource(path=install_path)
138+
name = install_path
141139

142140
# NOTE We can't reliably determine the package name from the lockfile v2 syntax,
143141
# but we need it for registry queries and special source processing;
@@ -208,7 +206,7 @@ def __init__(
208206
self.all_lockfiles: Set[Path] = set()
209207
# Mapping of lockfiles to a dict of the Git source target paths and GitSource objects.
210208
self.git_sources: DefaultDict[
211-
Path, Dict[Path, GitSource]
209+
Path, Dict[Path, Tuple[str, GitSource]]
212210
] = collections.defaultdict(lambda: {})
213211
# FIXME better pass the same provider object we created in main
214212
self.rcfile_provider = NpmRCFileProvider()
@@ -369,9 +367,49 @@ async def generate_package(self, package: Package) -> None:
369367
# Get a unique name to use for the Git repository folder.
370368
name = f'{package.name}-{source.commit}'
371369
path = self.gen.data_root / 'git-packages' / name
372-
self.git_sources[package.lockfile][path] = source
370+
self.git_sources[package.lockfile][path] = (package.name, source)
373371
self.gen.add_git_source(source.url, source.commit, path)
374372

373+
url = urllib.parse.urlparse(source.url)
374+
if url.netloc == '[email protected]' or url.netloc == 'github.com':
375+
url = url._replace(
376+
netloc='codeload.github.com', path=re.sub('.git$', '', url.path)
377+
)
378+
tarball_url = url._replace(
379+
path=re.sub('$', f'/tar.gz/{source.commit}', url.path)
380+
)
381+
index_url = tarball_url.geturl()
382+
elif url.netloc == '[email protected]' or url.netloc == 'gitlab.com':
383+
url = url._replace(
384+
netloc='gitlab.com', path=re.sub('.git$', '', url.path)
385+
)
386+
tarball_url = url._replace(
387+
path=re.sub(
388+
'$',
389+
f'/-/archive/{source.commit}/{package.name}-{source.commit}.tar.gz',
390+
url.path,
391+
)
392+
)
393+
index_url = url._replace(
394+
path=re.sub(
395+
'$', f'/repository/archive.tar.gz?ref={source.commit}', url.path
396+
)
397+
).geturl()
398+
metadata = await RemoteUrlMetadata.get(
399+
tarball_url.geturl(), cachable=True, integrity_algorithm='sha512'
400+
)
401+
402+
self.gen.add_url_source(
403+
url=tarball_url.geturl(),
404+
integrity=metadata.integrity,
405+
destination=self.get_cacache_content_path(metadata.integrity),
406+
)
407+
408+
self.add_index_entry(
409+
url=index_url,
410+
metadata=metadata,
411+
)
412+
375413
elif isinstance(source, LocalSource):
376414
pass
377415

@@ -432,8 +470,8 @@ def _finalize(self) -> None:
432470
if type == "object"
433471
then
434472
to_entries | map(
435-
if (.value | type == "string") and $data[.value]
436-
then .value = "git+file:\($buildroot)/\($data[.value])"
473+
if (.key | type == "string") and $data[.key]
474+
then .value = "git+file://\($buildroot)/\($data[.key])"
437475
else .
438476
end
439477
) | from_entries
@@ -445,7 +483,7 @@ def _finalize(self) -> None:
445483
walk(
446484
if type == "object" and (.version | type == "string") and $data[.version]
447485
then
448-
.version = "git+file:\($buildroot)/\($data[.version])"
486+
.resolved = "git+file:\($buildroot)/\($data[.version])"
449487
else .
450488
end
451489
)
@@ -459,56 +497,57 @@ def _finalize(self) -> None:
459497
'package-lock.json': {},
460498
}
461499

462-
for path, source in sources.items():
463-
GIT_URL_PREFIX = 'git+'
464-
465-
new_version = f'{path}#{source.commit}'
466-
assert source.from_ is not None
467-
data['package.json'][source.from_] = new_version
468-
data['package-lock.json'][source.original] = new_version
469-
470-
if source.from_.startswith(GIT_URL_PREFIX):
471-
data['package.json'][
472-
source.from_[len(GIT_URL_PREFIX) :]
473-
] = new_version
474-
475-
if source.original.startswith(GIT_URL_PREFIX):
476-
data['package-lock.json'][
477-
source.original[len(GIT_URL_PREFIX) :]
478-
] = new_version
479-
480-
for filename, script in scripts.items():
481-
target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename
482-
script = (
483-
textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '')
484-
)
485-
json_data = json.dumps(data[filename])
486-
patch_commands[lockfile].append(
487-
'jq'
488-
' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"'
489-
f' --argjson data {shlex.quote(json_data)}'
490-
f' {shlex.quote(script)} {target}'
491-
f' > {target}.new'
492-
)
493-
patch_commands[lockfile].append(f'mv {target}{{.new,}}')
494-
495-
patch_all_commands: List[str] = []
496-
for lockfile in self.all_lockfiles:
497-
patch_dest = (
498-
self.gen.data_root / 'patch' / self.relative_lockfile_dir(lockfile)
499-
)
500-
# Don't use with_extension to avoid problems if the package has a . in its name.
501-
patch_dest = patch_dest.with_name(patch_dest.name + '.sh')
500+
with open(lockfile) as fp:
501+
lockfile_v1 = json.load(fp)['lockfileVersion'] == 1
502+
503+
if lockfile_v1:
504+
for path, name_source in sources.items():
505+
GIT_URL_PREFIX = 'git+'
506+
name, source = name_source
507+
new_version = f'{path}#{source.commit}'
508+
data['package.json'][name] = new_version
509+
data['package-lock.json'][source.original] = new_version
510+
511+
if source.original.startswith(GIT_URL_PREFIX):
512+
data['package-lock.json'][
513+
source.original[len(GIT_URL_PREFIX) :]
514+
] = new_version
515+
516+
for filename, script in scripts.items():
517+
target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename
518+
script = (
519+
textwrap.dedent(script.lstrip('\n'))
520+
.strip()
521+
.replace('\n', '')
522+
)
523+
json_data = json.dumps(data[filename])
524+
patch_commands[lockfile].append(
525+
'jq'
526+
' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"'
527+
f' --argjson data {shlex.quote(json_data)}'
528+
f' {shlex.quote(script)} {target}'
529+
f' > {target}.new'
530+
)
531+
patch_commands[lockfile].append(f'mv {target}{{.new,}}')
532+
533+
if len(patch_commands) > 0:
534+
patch_all_commands: List[str] = []
535+
for lockfile in self.all_lockfiles:
536+
patch_dest = (
537+
self.gen.data_root / 'patch' / self.relative_lockfile_dir(lockfile)
538+
)
539+
# Don't use with_extension to avoid problems if the package has a . in its name.
540+
patch_dest = patch_dest.with_name(patch_dest.name + '.sh')
502541

503-
self.gen.add_script_source(patch_commands[lockfile], patch_dest)
504-
patch_all_commands.append(f'"$FLATPAK_BUILDER_BUILDDIR"/{patch_dest}')
542+
self.gen.add_script_source(patch_commands[lockfile], patch_dest)
543+
patch_all_commands.append(f'$FLATPAK_BUILDER_BUILDDIR/{patch_dest}')
505544

506-
patch_all_dest = self.gen.data_root / 'patch-all.sh'
507-
self.gen.add_script_source(patch_all_commands, patch_all_dest)
545+
patch_all_dest = self.gen.data_root / 'patch-all.sh'
546+
self.gen.add_script_source(patch_all_commands, patch_all_dest)
508547

509-
if not self.no_autopatch:
510-
# FLATPAK_BUILDER_BUILDDIR isn't defined yet for script sources.
511-
self.gen.add_command(f'FLATPAK_BUILDER_BUILDDIR="$PWD" {patch_all_dest}')
548+
if not self.no_autopatch:
549+
# FLATPAK_BUILDER_BUILDDIR isn't defined yet for script sources.
550+
self.gen.add_command(f'FLATPAK_BUILDER_BUILDDIR=$PWD {patch_all_dest}')
512551

513552
if self.index_entries:
514553
for path, entry in self.index_entries.items():
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
{
2+
"name": "@flatpak-node-generator-tests/git",
3+
"version": "1.0.0",
4+
"lockfileVersion": 1,
5+
"requires": true,
6+
"dependencies": {
7+
"@flatpak-node-generator-tests/subdir": {
8+
"version": "file:subdir",
9+
"requires": {
10+
"arr-diff": "github:jonschlinkert/arr-diff#f39e5f10f55fd0cc38fa686c6ebc11747815850c",
11+
"array.chunk": "git+https://github.com/zhiyelee/array.chunk.git",
12+
"decamelize": "git+https://github.com/sindresorhus/decamelize.git#da8552d335f9067354ca9b93378482eb9f1fb821",
13+
"filled-array": "github:sindresorhus/filled-array#v2.1.0",
14+
"in-array": "git+ssh://[email protected]/jonschlinkert/in-array.git",
15+
"sorted-object": "github:domenic/sorted-object",
16+
"text-encoding-shim": "gitlab:t-affeldt/text-encoding-shim",
17+
"unordered-array-remove": "git+ssh://[email protected]/mafintosh/unordered-array-remove.git#9ecfd7d63da6e5499b3a530574aca2d8c826af93"
18+
}
19+
},
20+
"arr-diff": {
21+
"version": "github:jonschlinkert/arr-diff#f39e5f10f55fd0cc38fa686c6ebc11747815850c",
22+
"from": "github:jonschlinkert/arr-diff#f39e5f10f55fd0cc38fa686c6ebc11747815850c"
23+
},
24+
"array-range": {
25+
"version": "github:mattdesl/array-range#d8b6fbdfc29caf846be02229325dfb4967f1dcd6",
26+
"from": "github:mattdesl/array-range#master"
27+
},
28+
"array.chunk": {
29+
"version": "git+https://github.com/zhiyelee/array.chunk.git#1ba8011a64448210e334a17642e395690a7164c0",
30+
"from": "git+https://github.com/zhiyelee/array.chunk.git"
31+
},
32+
"decamelize": {
33+
"version": "git+https://github.com/sindresorhus/decamelize.git#da8552d335f9067354ca9b93378482eb9f1fb821",
34+
"from": "git+https://github.com/sindresorhus/decamelize.git#da8552d335f9067354ca9b93378482eb9f1fb821"
35+
},
36+
"filled-array": {
37+
"version": "github:sindresorhus/filled-array#3529bc985247d0f84db4080fecd8276643838d0c",
38+
"from": "github:sindresorhus/filled-array#v2.1.0"
39+
},
40+
"in-array": {
41+
"version": "git+ssh://[email protected]/jonschlinkert/in-array.git#47a5e55362098646b56a3ec6775bd5198df1c7ed",
42+
"from": "git+ssh://[email protected]/jonschlinkert/in-array.git"
43+
},
44+
"is-empty-object": {
45+
"version": "github:gummesson/is-empty-object#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8",
46+
"from": "github:gummesson/is-empty-object#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8"
47+
},
48+
"is-number": {
49+
"version": "github:jonschlinkert/is-number#98e8ff1da1a89f93d1397a24d7413ed15421c139",
50+
"from": "github:jonschlinkert/is-number"
51+
},
52+
"person-lib": {
53+
"version": "gitlab:volodymyrkr/person-lib#752fd1828b1eb3a9635bf725ae5e1704a375e524",
54+
"from": "gitlab:volodymyrkr/person-lib"
55+
},
56+
"sorted-object": {
57+
"version": "github:domenic/sorted-object#87105deb13d4f4151b2abd1a78d27a5216e3e79d",
58+
"from": "github:domenic/sorted-object"
59+
},
60+
"text-encoding-shim": {
61+
"version": "gitlab:t-affeldt/text-encoding-shim#33b05934b4e4e6c65fc06260eaa3b2aa2909e7b5",
62+
"from": "gitlab:t-affeldt/text-encoding-shim"
63+
},
64+
"to-camel-case": {
65+
"version": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
66+
"from": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git",
67+
"requires": {
68+
"to-space-case": "^1.0.0"
69+
}
70+
},
71+
"to-capital-case": {
72+
"version": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
73+
"from": "git+https://[email protected]/ianstormtaylor/to-capital-case.git",
74+
"requires": {
75+
"to-space-case": "^1.0.0"
76+
}
77+
},
78+
"to-no-case": {
79+
"version": "1.0.0",
80+
"resolved": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90"
81+
},
82+
"to-space-case": {
83+
"version": "1.0.0",
84+
"resolved": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
85+
"requires": {
86+
"to-no-case": "^1.0.0"
87+
}
88+
},
89+
"unordered-array-remove": {
90+
"version": "git+ssh://[email protected]/mafintosh/unordered-array-remove.git#9ecfd7d63da6e5499b3a530574aca2d8c826af93",
91+
"from": "git+ssh://[email protected]/mafintosh/unordered-array-remove.git#9ecfd7d63da6e5499b3a530574aca2d8c826af93"
92+
}
93+
}
94+
}

0 commit comments

Comments
 (0)