Skip to content

Commit

Permalink
style
Browse files Browse the repository at this point in the history
  • Loading branch information
haampie committed May 7, 2024
1 parent 6ff59f2 commit 4597fe0
Show file tree
Hide file tree
Showing 8 changed files with 119 additions and 140 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/clingo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
spack config add config:install_tree:padded_length:256
spack config add "packages:all:require:[target=$(cat target.txt)]"
spack config blame
- name: Install clingo
- name: install
run: |
. spack/share/spack/setup-env.sh
spack python clingo/scripts/install_clingo.py
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/gnupg.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: "GnuPG"
name: gnupg

on:
push:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/patchelf.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: "Patchelf"
name: patchelf

on:
push:
Expand Down
96 changes: 43 additions & 53 deletions clingo/scripts/clingo_json.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
"""Produce the clingo.json file associated with the mirror"""
import json

import glob
import hashlib
import os.path

import json
import os.path

# Each entry in clingo.json has the following keys:
#
Expand All @@ -17,42 +16,35 @@

# Dictionary that maps (OS, TARGET) to info for the spec
SPEC_INFO = {
('rhel5', 'x86_64'): {
'spec': 'clingo-bootstrap%gcc platform=linux target=x86_64',
},
('centos7', 'x86_64'): {
'spec': 'clingo-bootstrap%gcc platform=linux target=x86_64',
},
('centos7', 'aarch64'): {
'spec': 'clingo-bootstrap%gcc platform=linux target=aarch64',
},
('centos7', 'ppc64le'): {
'spec': 'clingo-bootstrap%gcc platform=linux target=ppc64le',
},
('monterey', 'x86_64'): {
'spec': 'clingo-bootstrap%apple-clang platform=darwin target=x86_64',
("rhel5", "x86_64"): {"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"},
("centos7", "x86_64"): {"spec": "clingo-bootstrap%gcc platform=linux target=x86_64"},
("centos7", "aarch64"): {"spec": "clingo-bootstrap%gcc platform=linux target=aarch64"},
("centos7", "ppc64le"): {"spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"},
("monterey", "x86_64"): {"spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"},
("ventura", "aarch64"): {
"spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
},
('ventura', 'aarch64'): {
'spec': 'clingo-bootstrap%apple-clang platform=darwin target=aarch64',
}
}


def sha256(path):
fn = hashlib.sha256()
with open(path, "rb") as f:
fn.update(f.read())
return fn.hexdigest()


def tarball_hash(path):
filename = os.path.basename(path)
filename = filename.replace('.spack', '')
return filename.split('-')[-1]
filename = filename.replace(".spack", "")
return filename.split("-")[-1]

shaglob_expr = './build_cache/**/*.spack'

shaglob_expr = "./build_cache/**/*.spack"
tarballs = glob.glob(shaglob_expr, recursive=True)
shas = {tarball_hash(tarball): sha256(tarball) for tarball in tarballs}

glob_expr = './build_cache/*.json'
glob_expr = "./build_cache/*.json"
spec_yaml_files = glob.glob(glob_expr)

mirror_info = []
Expand All @@ -62,46 +54,44 @@ def tarball_hash(path):

# Get the raw data from spec.json
with open(spec_json) as f:
spec_yaml_data = json.load(f)['spec']['nodes']
spec_yaml_data = json.load(f)["spec"]["nodes"]

# Cycle through the specs in raw data. We are only interested
# Cycle through the specs in raw data. We are only interested
# in clingo bootstrap
binary_data = {}
for entry in spec_yaml_data:
current_spec = entry['name']
if current_spec not in ('clingo-bootstrap', 'python'):
current_spec = entry["name"]
if current_spec not in ("clingo-bootstrap", "python"):
continue

if current_spec == 'clingo-bootstrap':
if current_spec == "clingo-bootstrap":
clingo_data = entry
binary_data['clingo'] = clingo_data
binary_data["clingo"] = clingo_data

elif current_spec == 'python':
elif current_spec == "python":
python_data = entry
binary_data['python'] = python_data
assert 'clingo' in binary_data, 'entry "clingo" is required'
assert 'python' in binary_data, 'entry "python" is required'
current_os = binary_data['clingo']['arch']['platform_os']
current_target = binary_data['clingo']['arch']['target']
compiler_name = binary_data['clingo']['compiler']['name']
compiler_version = str(binary_data['clingo']['compiler']['version'])

python_version = binary_data['python']['version']
python_spec = 'python@{0}'.format(python_version)

current_hash = binary_data['clingo']['hash']
binary_data["python"] = python_data

assert "clingo" in binary_data, 'entry "clingo" is required'
assert "python" in binary_data, 'entry "python" is required'

current_os = binary_data["clingo"]["arch"]["platform_os"]
current_target = binary_data["clingo"]["arch"]["target"]

compiler_name = binary_data["clingo"]["compiler"]["name"]
compiler_version = str(binary_data["clingo"]["compiler"]["version"])

python_version = binary_data["python"]["version"]
python_spec = "python@{0}".format(python_version)

current_hash = binary_data["clingo"]["hash"]
mirror_entry = {
"spec": SPEC_INFO[(current_os, current_target)]['spec'],
"spec": SPEC_INFO[(current_os, current_target)]["spec"],
"python": python_spec,
"binaries": [
('clingo-bootstrap', current_hash, shas[current_hash])
],
"binaries": [("clingo-bootstrap", current_hash, shas[current_hash])],
}
mirror_info.append(mirror_entry)

mirror_info = sorted(mirror_info, key=lambda x: (x['spec'], x['python']))
with open('./clingo.json', 'w') as f:
json.dump({'verified': mirror_info}, f, sort_keys=True, indent=2)
mirror_info = sorted(mirror_info, key=lambda x: (x["spec"], x["python"]))
with open("./clingo.json", "w") as f:
json.dump({"verified": mirror_info}, f, sort_keys=True, indent=2)
5 changes: 2 additions & 3 deletions clingo/scripts/install_clingo.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,15 @@
import spack.bootstrap.config
import spack.main


if sys.platform == "linux":
CLINGO_BASE_SPEC = "clingo-bootstrap@spack +static_libstdcpp +optimized +ipo ~docs"
else:
CLINGO_BASE_SPEC = "clingo-bootstrap@spack +optimized +ipo ~docs"

install = spack.main.SpackCommand('install')
install = spack.main.SpackCommand("install")


with spack.bootstrap.config.spack_python_interpreter():
msg = 'Installing clingo-bootstrap with Python: {0}'
msg = "Installing clingo-bootstrap with Python: {0}"
print(msg.format(spack.bootstrap.config.spec_for_current_python()))
install(CLINGO_BASE_SPEC)
81 changes: 35 additions & 46 deletions gnupg/scripts/gnupg_json.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""Produce the gnupg.json file associated with the mirror"""
import json

import glob
import hashlib
import json
Expand All @@ -8,7 +8,6 @@
import spack.spec
import spack.traverse


# Each entry in gnupg.json has the following keys:
#
# "spec": root spec to be matched
Expand All @@ -19,82 +18,72 @@

# Dictionary that maps (OS, TARGET) to info for the spec
SPEC_INFO = {
('centos7', 'x86_64'): {
'spec': '[email protected]: %gcc platform=linux target=x86_64',
},
('centos7', 'aarch64'): {
'spec': '[email protected]: %gcc platform=linux target=aarch64',
},
('centos7', 'ppc64le'): {
'spec': '[email protected]: %gcc platform=linux target=ppc64le',
},
('monterey', 'x86_64'): {
'spec': '[email protected]: %apple-clang platform=darwin target=x86_64',
},
('sonoma', 'aarch64'): {
'spec': '[email protected]: %apple-clang platform=darwin target=aarch64',
}
("centos7", "x86_64"): {"spec": "[email protected]: %gcc platform=linux target=x86_64"},
("centos7", "aarch64"): {"spec": "[email protected]: %gcc platform=linux target=aarch64"},
("centos7", "ppc64le"): {"spec": "[email protected]: %gcc platform=linux target=ppc64le"},
("monterey", "x86_64"): {"spec": "[email protected]: %apple-clang platform=darwin target=x86_64"},
("sonoma", "aarch64"): {"spec": "[email protected]: %apple-clang platform=darwin target=aarch64"},
}


def sha256(path):
fn = hashlib.sha256()
with open(path, "rb") as f:
fn.update(f.read())
return fn.hexdigest()


def tarball_hash(path):
filename = os.path.basename(path)
filename = filename.replace('.spack', '')
return filename.split('-')[-1]
filename = filename.replace(".spack", "")
return filename.split("-")[-1]

shaglob_expr = './build_cache/**/*.spack'

shaglob_expr = "./build_cache/**/*.spack"
tarballs = glob.glob(shaglob_expr, recursive=True)
shas = {tarball_hash(tarball): sha256(tarball) for tarball in tarballs}

glob_expr = './build_cache/*.json'
glob_expr = "./build_cache/*.json"
spec_json_files = glob.glob(glob_expr)

mirror_info = []
spec_json_dict = {}
for spec_json in spec_json_files:
if 'gnupg' not in spec_json:
if "gnupg" not in spec_json:
continue

s = spack.spec.Spec.from_specfile(spec_json)
binaries = []
for edge in reversed(spack.traverse.traverse_edges_topo([s], direction="children", deptype=("link", "run"))):
node = edge.spec
binaries.append(
(node.name, node.dag_hash(), shas[node.dag_hash()])
)
for edge in reversed(
spack.traverse.traverse_edges_topo([s], direction="children", deptype=("link", "run"))
):
node = edge.spec
binaries.append((node.name, node.dag_hash(), shas[node.dag_hash()]))

# Get the raw data from spec.json
with open(spec_json) as f:
spec_json_data = json.load(f)['spec']['nodes']
spec_json_data = json.load(f)["spec"]["nodes"]

# Find the GnuPG entry and store it somewhere
binary_data = {}
for entry in spec_json_data:
if 'gnupg' == entry['name']:
binary_data['gnupg'] = entry
assert 'gnupg' in binary_data
if "gnupg" == entry["name"]:
binary_data["gnupg"] = entry
assert "gnupg" in binary_data

assert 'gnupg' in binary_data, 'entry "gnupg" is required'
current_os = binary_data['gnupg']['arch']['platform_os']
current_target = binary_data['gnupg']['arch']['target']
assert "gnupg" in binary_data, 'entry "gnupg" is required'

current_os = binary_data["gnupg"]["arch"]["platform_os"]
current_target = binary_data["gnupg"]["arch"]["target"]

# If the target is not generic, like x86_64 etc. it's a fully fledged object
if not isinstance(current_target, str):
current_target = current_target["name"]

current_hash = binary_data['gnupg']['hash']
mirror_entry = {
"spec": SPEC_INFO[(current_os, current_target)]['spec'],
"binaries": binaries,
}

current_hash = binary_data["gnupg"]["hash"]
mirror_entry = {"spec": SPEC_INFO[(current_os, current_target)]["spec"], "binaries": binaries}
mirror_info.append(mirror_entry)

mirror_info = sorted(mirror_info, key=lambda x: x['spec'])
with open('./gnupg.json', 'w') as f:
json.dump({'verified': mirror_info}, f, sort_keys=True, indent=2)
mirror_info = sorted(mirror_info, key=lambda x: x["spec"])
with open("./gnupg.json", "w") as f:
json.dump({"verified": mirror_info}, f, sort_keys=True, indent=2)
Loading

0 comments on commit 4597fe0

Please sign in to comment.