From b52ccc26f27d18f14f8de59a82d774233fe3eaa9 Mon Sep 17 00:00:00 2001 From: Eric Douglass Date: Mon, 14 May 2018 09:04:46 -0400 Subject: [PATCH 1/5] saving off the paint metadata changes --- bin/validate.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/bin/validate.py b/bin/validate.py index c186646a..75a802ef 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -93,14 +93,13 @@ def download_source_gafs(group_metadata, target_dir, exclusions=[]): return downloaded_paths def check_and_download_paint_source(paint_metadata, group_id, dataset, target_dir): - paint_dataset = find(paint_metadata["datasets"], lambda d: d["dataset"] == "paint_{}".format(dataset)) - if paint_dataset is None: + if paint_metadata is None: return None - path = os.path.join(target_dir, "groups", group_id, "{}-src.gaf.gz".format(paint_dataset["dataset"])) + path = os.path.join(target_dir, "groups", group_id, "{}-src.gaf.gz".format(paint_metadata["dataset"])) click.echo("Downloading paint to {}".format(path)) - urllib.request.urlretrieve(paint_dataset["source"], path) - unzipped = os.path.join(os.path.split(path)[0], "{}-src.gaf".format(paint_dataset["dataset"])) + urllib.request.urlretrieve(paint_metadata["source"], path) + unzipped = os.path.join(os.path.split(path)[0], "{}-src.gaf".format(paint_metadata["dataset"])) unzip(path, unzipped) return unzipped @@ -325,20 +324,26 @@ def produce(group, metadata, gpad, ttl, target, ontology, exclude): click.echo("Loading ontology: {}...".format(ontology)) ontology_graph = OntologyFactory().create(ontology) + # Dictionary from dataset name to path in the target directory to the source zip. source_gaf_zips = download_source_gafs(group_metadata, absolute_target, exclusions=exclude) source_gafs = {zip_path: os.path.join(os.path.split(zip_path)[0], "{}-src.gaf".format(dataset)) for dataset, zip_path in source_gaf_zips.items()} for source_zip, source_gaf in source_gafs.items(): unzip(source_zip, source_gaf) - paint_metadata = metadata_file(absolute_metadata, "paint") + for metadataset in group_metadata["datasets"]: + dataset = metadataset["dataset"] + if not dataset in source_gaf_zips: + click.echo("Skipping {}".format(dataset)) + continue - for dataset in source_gaf_zips.keys(): gafzip = source_gaf_zips[dataset] source_gaf = source_gafs[gafzip] valid_gaf = produce_gaf(dataset, source_gaf, ontology_graph)[0] gpi = produce_gpi(dataset, absolute_target, valid_gaf, ontology_graph) + paint_metadata = metadataset.get("paint", None) + paint_src_gaf = check_and_download_paint_source(paint_metadata, group_metadata["id"], dataset, absolute_target) end_gaf = valid_gaf From 02dc1fa103dacde9165409b7764397ed1a4c4651 Mon Sep 17 00:00:00 2001 From: Eric Douglass Date: Wed, 23 May 2018 16:21:00 -0700 Subject: [PATCH 2/5] bumping minor version --- Makefile | 4 ++++ ontobio/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bf7aa305..a8dc0df9 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,10 @@ release: cleandist python setup.py sdist bdist_wheel bdist_egg twine upload dist/* +dev-release: cleandist + python setup.py sdist bdist_wheel bdist_egg + twine upload dist/* + nb: PYTHONPATH=.. jupyter notebook diff --git a/ontobio/__init__.py b/ontobio/__init__.py index a37a2c42..61e14d2c 100644 --- a/ontobio/__init__.py +++ b/ontobio/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -__version__ = '1.0.4' +__version__ = '1.1.0' from .ontol_factory import OntologyFactory from .ontol import Ontology, Synonym, TextDefinition From 4b53044464849dc25d952e546571f0f8f45725c9 Mon Sep 17 00:00:00 2001 From: Eric Douglass Date: Wed, 23 May 2018 16:51:32 -0700 Subject: [PATCH 3/5] updating ontobio version in test pypi --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index a8dc0df9..bf7aa305 100644 --- a/Makefile +++ b/Makefile @@ -31,10 +31,6 @@ release: cleandist python setup.py sdist bdist_wheel bdist_egg twine upload dist/* -dev-release: cleandist - python setup.py sdist bdist_wheel bdist_egg - twine upload dist/* - nb: PYTHONPATH=.. jupyter notebook From ae4fc4228abb0fbeb5567268a2f40dfcd1073c2c Mon Sep 17 00:00:00 2001 From: Eric Douglass Date: Fri, 25 May 2018 19:13:19 -0700 Subject: [PATCH 4/5] fixing a metadata processing bug --- bin/validate.py | 2 +- ontobio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/validate.py b/bin/validate.py index 58e4102c..110d152e 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -336,7 +336,7 @@ def produce(group, metadata, gpad, ttl, target, ontology, exclude): for metadataset in group_metadata["datasets"]: dataset = metadataset["dataset"] - if not dataset in source_gaf_zips: + if metadataset["type"] != "gaf": click.echo("Skipping {}".format(dataset)) continue diff --git a/ontobio/__init__.py b/ontobio/__init__.py index 61e14d2c..a3645825 100644 --- a/ontobio/__init__.py +++ b/ontobio/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -__version__ = '1.1.0' +__version__ = '1.2.0' from .ontol_factory import OntologyFactory from .ontol import Ontology, Synonym, TextDefinition From 86d977d478ffe5b5a2ade05a69a89227ca609f4e Mon Sep 17 00:00:00 2001 From: Eric Douglass Date: Tue, 29 May 2018 10:38:41 -0700 Subject: [PATCH 5/5] skipping non-downloaded sources correctly and version update --- bin/validate.py | 5 +++-- ontobio/__init__.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/validate.py b/bin/validate.py index 110d152e..0e15cb4f 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -336,8 +336,9 @@ def produce(group, metadata, gpad, ttl, target, ontology, exclude): for metadataset in group_metadata["datasets"]: dataset = metadataset["dataset"] - if metadataset["type"] != "gaf": - click.echo("Skipping {}".format(dataset)) + # Skip this dataset if it's not gaf, or if it's not something we've downloaded + if metadataset["type"] != "gaf" or dataset not in source_gaf_zips: + click.echo("Skipping {}".format(metadataset["id"])) continue gafzip = source_gaf_zips[dataset] diff --git a/ontobio/__init__.py b/ontobio/__init__.py index a3645825..1919a9c4 100644 --- a/ontobio/__init__.py +++ b/ontobio/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -__version__ = '1.2.0' +__version__ = '1.2.1' from .ontol_factory import OntologyFactory from .ontol import Ontology, Synonym, TextDefinition