From 7924da3d846eaa9ce102cc3bf8f795b54e1e94d0 Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Sat, 6 Jun 2020 15:11:39 +1200 Subject: [PATCH 01/10] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BB=20Makin?= =?UTF-8?q?g=20@id=20tag=20required=20for=20document?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests are going to crap themselves now --- schema.igsn.org/json/registration/v0.1/main.schema.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schema.igsn.org/json/registration/v0.1/main.schema.json b/schema.igsn.org/json/registration/v0.1/main.schema.json index 7da61c1..78b0981 100644 --- a/schema.igsn.org/json/registration/v0.1/main.schema.json +++ b/schema.igsn.org/json/registration/v0.1/main.schema.json @@ -8,6 +8,10 @@ "@context": { "const": "https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld" }, + "@id": { + "type": "string", + "description": "An identifier for this JSON document (e.g. 'http://example-agent.org/igsn/id/XXXAGENT00234')" + }, "igsn": { "$ref": "identifiers.schema.json#/definitions/igsnPid" }, @@ -56,5 +60,5 @@ }, "log": { "$ref": "eventLog.schema.json" } }, - "required": ["@context", "igsn", "registrant"] + "required": ["@context", "@id", "igsn", "registrant"] } From ba29f92e14c82af5610b465f9400ef47bba3a16d Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Sat, 6 Jun 2020 15:13:40 +1200 Subject: [PATCH 02/10] =?UTF-8?q?=F0=9F=A7=AA=20Injecting=20@id=20tags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @id tags now required in registration schema --- tests/test_registration_schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_registration_schema.py b/tests/test_registration_schema.py index 3af0d15..da0b6f2 100644 --- a/tests/test_registration_schema.py +++ b/tests/test_registration_schema.py @@ -81,6 +81,7 @@ def test_igsn_registration( # Inject required IGSN registration context location = f"{schema_home}/{profiles['registration']}" obj["@context"] = f"{location}/context.jsonld" + obj['@id'] = "https://example-agent.org/igsns/XXXTEST234" # Check validation (or not...) registration_validator(obj) From a0681eba4ab489f5ada025a4af6c80f87bc46b6f Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Sat, 6 Jun 2020 15:25:47 +1200 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=A7=AA=20Change=20JSON-LD=20context?= =?UTF-8?q?=20injection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now we point to the master version so that branches don't randomly fail on schema checks because the contstant value is wrong... To fix properly we will need to do schema generation from templates, but this is a job for another day --- tests/conftest.py | 3 +-- tests/test_registration_schema.py | 12 +++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4a76fae..a1713b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,11 +29,10 @@ def schema_home(): "host": "https://raw.githubusercontent.com", "org": "IGSN", "repo": "igsn-json", - "branch": "master", + "branch": "issue-9-Merge_in_context_jsonld_to_registration_schema", } return f"{github['host']}/{github['org']}/{github['repo']}/{github['branch']}" - @pytest.fixture() def registration_schema_folder(profiles): return ROOT / profiles["registration"] diff --git a/tests/test_registration_schema.py b/tests/test_registration_schema.py index da0b6f2..3b1fa94 100644 --- a/tests/test_registration_schema.py +++ b/tests/test_registration_schema.py @@ -2,6 +2,9 @@ import pytest from fastjsonschema import JsonSchemaException +# Which context we should point to +JSONLD_CONTEXT = "https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld" + # Some bogus data for testing - as (obj, is_valid) tuples TEST_OBJECTS = [ ( @@ -71,7 +74,6 @@ ), ] - @pytest.mark.parametrize("obj,should_pass", TEST_OBJECTS) def test_igsn_registration( schema_home, profiles, registration_validator, obj, should_pass @@ -80,7 +82,7 @@ def test_igsn_registration( try: # Inject required IGSN registration context location = f"{schema_home}/{profiles['registration']}" - obj["@context"] = f"{location}/context.jsonld" + obj["@context"] = JSONLD_CONTEXT obj['@id'] = "https://example-agent.org/igsns/XXXTEST234" # Check validation (or not...) @@ -96,13 +98,13 @@ def test_igsn_registration( @pytest.mark.parametrize("obj,should_pass", TEST_OBJECTS) def test_against_remote_schema( - schema_home, profiles, remote_registration_validator, obj, should_pass + remote_registration_validator, obj, should_pass ): "Check against remote schema for the branch" try: # Inject required IGSN registration context - location = f"{schema_home}/{profiles['registration']}" - obj["@context"] = f"{location}/context.jsonld" + obj["@context"] = JSONLD_CONTEXT + obj["@id"] = "https://example-agent.org/igsns/XXXTEST234" # Check validation (or not...) remote_registration_validator(obj) From 98241f6eca0aa10c814632d8569bb8ec0705ebe8 Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Sat, 6 Jun 2020 15:26:14 +1200 Subject: [PATCH 04/10] =?UTF-8?q?=E2=9C=8D=20Updated=20examples=20to=20sat?= =?UTF-8?q?isfy=20new=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now include @id tags as required --- examples/full_registration.json | Bin 6584 -> 6696 bytes examples/full_registration.yaml | 3 +++ examples/minimal_registration.json | Bin 1190 -> 1302 bytes examples/minimal_registration.yaml | 3 +++ 4 files changed, 6 insertions(+) diff --git a/examples/full_registration.json b/examples/full_registration.json index 633877353a7dfaa27be70390310b9d033ba88b67..bcaa18248fc2be240701cd67cd0c93e9219f61c0 100644 GIT binary patch delta 55 zcmdmCyuxII7$d6#LncGYL`BKT9xMWq`V2`7`3&g{r3}Rk`ap?vAe{$fMNFQ_p}tv! Haft)~cfJh? delta 12 TcmZ2svcq_T7~|$P#zhhUAUOns diff --git a/examples/full_registration.yaml b/examples/full_registration.yaml index 7d96426..8e147b4 100644 --- a/examples/full_registration.yaml +++ b/examples/full_registration.yaml @@ -10,6 +10,9 @@ # IGSN data to be valid. "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +# The ID tag identifies this document +"@id": https://csiro.au/bogus/igsn/XXXCSIRO0001 + # This is the identifier that we are talking about. All identifier objects # have a 'kind' key indicating what sort of identifier they are, and an id # key pointing to a URI or resource definition diff --git a/examples/minimal_registration.json b/examples/minimal_registration.json index bad35eef04a315e98cb5697f5327e3e4cabe20b6..69020dc1f25f10eeeea6b55970b2cf15890e69d1 100644 GIT binary patch delta 53 zcmZ3+IgM+B7$d6#LncGYL`BKT9xMXl`V2`7`3&g{r3}Rk`ap?vAf3mcKUtMgd$Snh G1ZDt4ZVQ Date: Sat, 6 Jun 2020 15:35:02 +1200 Subject: [PATCH 05/10] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BB=20Updat?= =?UTF-8?q?e=20examples=20to=20meet=20new=20schema=20reqs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added @id tags for everything --- examples/lifecycle_1.json | Bin 7390 -> 7614 bytes examples/lifecycle_1.yaml | 2 ++ examples/lifecycle_2.json | Bin 9380 -> 9604 bytes examples/lifecycle_2.yaml | 2 ++ examples/lifecycle_3.json | Bin 17014 -> 17358 bytes examples/lifecycle_3.yaml | 3 +++ 6 files changed, 7 insertions(+) diff --git a/examples/lifecycle_1.json b/examples/lifecycle_1.json index d27ccf0defa506d3e0eadd6a8d8f04fb04baa365..de13b38b5208e55e7dfd88e64333432d9ecce513 100644 GIT binary patch delta 73 zcmca-xzBoo2_vJ!wSB$*z>% delta 17 YcmZqiUgEjIgmLo{Mg_smCc*(i06EkJhX4Qo diff --git a/examples/lifecycle_2.yaml b/examples/lifecycle_2.yaml index c3b382d..f91b3eb 100644 --- a/examples/lifecycle_2.yaml +++ b/examples/lifecycle_2.yaml @@ -7,6 +7,7 @@ --- # tooth "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +"@id": "https://mfo.org/specimens/XXXMFO0001" igsn: XXXMFO0001 @@ -61,6 +62,7 @@ log: # Registration of the sediment horizon "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +"@id": "https://mfo.org/specimens/XXXMFO0002" igsn: XXXMFO0002 diff --git a/examples/lifecycle_3.json b/examples/lifecycle_3.json index b6817bbf9a9e4e58b4201fd267f5aacf836eaed3..896c177f6614511a0016f63beb48402d79186e88 100644 GIT binary patch delta 124 zcmey?!g#Koaf1mXqr>F)f})cZSUD!YV&a>;jzes+3!~oTI9;*H>$rF}hcVVkU{SJy zOK0){W|qkbx*|YjYMTY5{Dh&(0`w&&I~Z{Y>M|5FBr@bO6fop4qypLclXGW^dv+9 D>HQI~ diff --git a/examples/lifecycle_3.yaml b/examples/lifecycle_3.yaml index 3cae0a6..7c55693 100644 --- a/examples/lifecycle_3.yaml +++ b/examples/lifecycle_3.yaml @@ -7,6 +7,7 @@ --- # tooth "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +"@id": "https://mfo.org/specimens/XXXMFO0001" igsn: XXXMFO0001 @@ -82,6 +83,7 @@ log: # Registration of the sediment horizon "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +"@id": "https://mfo.org/specimens/XXXMFO0002" igsn: XXXMFO0002 @@ -134,6 +136,7 @@ log: --- # tooth sample for CSIRO "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +"@id": "https://csiro.au/lab-samples/XXXCSIRO001" igsn: XXXCSIRO001 From b05b4e9b943888af7b33be30ee57e95c1718c962 Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Sat, 6 Jun 2020 15:36:30 +1200 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=92=A9=20WIP=20updating=20examples.?= =?UTF-8?q?md=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/examples.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/examples.md b/examples/examples.md index e7b6648..6ab761f 100644 --- a/examples/examples.md +++ b/examples/examples.md @@ -59,6 +59,7 @@ This is the minimal set of metadata required to define an IGSN - the `registrant ```yaml "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +"@id": https://csiro.au/bogus_igsns/XXXCSIRO0001 igsn: XXXCSIRO0001 registrant: name: CSIRO From ffcb368a93962782d6242f8823a2958cca31d153 Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Sat, 6 Jun 2020 16:49:42 +1200 Subject: [PATCH 07/10] =?UTF-8?q?=F0=9F=A7=B9Delinting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index a1713b5..84454d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,7 @@ def schema_home(): } return f"{github['host']}/{github['org']}/{github['repo']}/{github['branch']}" + @pytest.fixture() def registration_schema_folder(profiles): return ROOT / profiles["registration"] From 6af30f014fe9b78ce26bd3e6fa808cd59c0f1ef0 Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Sat, 6 Jun 2020 16:54:46 +1200 Subject: [PATCH 08/10] =?UTF-8?q?=E2=9C=8D=20Finishing=20updates=20to=20ex?= =?UTF-8?q?amples.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/examples.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/examples.md b/examples/examples.md index 6ab761f..e06f573 100644 --- a/examples/examples.md +++ b/examples/examples.md @@ -10,6 +10,7 @@ We provide the examples in YAML so that they're easier to read (and because JSON $ python convert.py minimal_registration.yaml { "@context": "https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld", + "@id": "https://csiro.au/bogus_igsn/XXXCSIRO0001", "igsn": "XXXCSIRO0001", "registrant": { "name": "CSIRO", @@ -55,7 +56,7 @@ Let's walk through the examples we've got... ## Minimal registration -This is the minimal set of metadata required to define an IGSN - the `registrant` field pointing to the allocating agent, and the identifier itself. Not very useful but easy to do: +This is the minimal set of metadata required to define an IGSN - the `registrant` field pointing to the allocating agent, an `@id` tag that identifies the JSON document, and the identifier itself. Not very useful but easy to do: ```yaml "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld @@ -169,6 +170,7 @@ We'll say that an archeology team from ANU, in collaboration with the Museum of ```yaml "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +@id: https://mfo.org/specimens/XXXMFO0001 igsn: XXXMFO0001 registrant: name: Museum of Fictional Objects @@ -206,6 +208,7 @@ The team have also sampled the stratigraphic horizon that this tooth was found i ```yaml "@context": https://raw.githubusercontent.com/IGSN/igsn-json/master/schema.igsn.org/json/registration/v0.1/context.jsonld +@id: https://mfo.org/specimens/XXXMFO0002 igsn: XXXMFO0002 registrant: $ref: http://igsn.org/XXXMFO001#registrant From 6e138c63e0f94d4e340c31e2aa720c0573bba4d9 Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Tue, 9 Jun 2020 11:05:40 +1200 Subject: [PATCH 09/10] =?UTF-8?q?=F0=9F=93=A6=20Add=20pyld=20to=20dependen?= =?UTF-8?q?cies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Pipfile | 1 + Pipfile.lock | 124 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 91 insertions(+), 34 deletions(-) diff --git a/Pipfile b/Pipfile index 35eff35..c2d4962 100644 --- a/Pipfile +++ b/Pipfile @@ -14,6 +14,7 @@ fastjsonschema = "*" jsonref = "*" pyyaml = "*" click = "*" +pyld = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 132bd46..fc09029 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,10 +1,12 @@ { "_meta": { "hash": { - "sha256": "6e202cf79b57d07d9ece5bd17edf951928cc460019e0052efad2c859997a16f7" + "sha256": "9b6537c64b2f9adf781497041b4844cc25d22a9a7e746bffb79e40e2229622f6" }, "pipfile-spec": 6, - "requires": {}, + "requires": { + "python_version": "3.8" + }, "sources": [ { "name": "pypi", @@ -14,12 +16,20 @@ ] }, "default": { + "cachetools": { + "hashes": [ + "sha256:1d057645db16ca7fe1f3bd953558897603d6f0b9c51ed9d11eb4d071ec4e2aab", + "sha256:de5d88f87781602201cde465d3afe837546663b168e8b39df67411b0bf10cefc" + ], + "markers": "python_version ~= '3.5'", + "version": "==4.1.0" + }, "certifi": { "hashes": [ - "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", - "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" + "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1", + "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc" ], - "version": "==2020.4.5.1" + "version": "==2020.4.5.2" }, "chardet": { "hashes": [ @@ -44,6 +54,12 @@ "index": "pypi", "version": "==2.14.4" }, + "frozendict": { + "hashes": [ + "sha256:774179f22db2ef8a106e9c38d4d1f8503864603db08de2e33be5b778230f6e45" + ], + "version": "==1.2" + }, "idna": { "hashes": [ "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", @@ -60,6 +76,46 @@ "index": "pypi", "version": "==0.2" }, + "lxml": { + "hashes": [ + "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", + "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", + "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", + "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", + "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", + "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", + "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", + "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", + "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", + "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", + "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", + "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", + "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", + "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", + "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", + "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", + "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", + "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", + "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", + "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", + "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", + "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", + "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", + "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", + "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", + "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", + "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.5.1" + }, + "pyld": { + "hashes": [ + "sha256:d3ac99b9cadeebefbf032ba9951b66bb96438a237273a6c6aead8c15c7564ea9" + ], + "index": "pypi", + "version": "==2.0.2" + }, "pyyaml": { "hashes": [ "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", @@ -222,37 +278,37 @@ }, "pytest": { "hashes": [ - "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3", - "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698" + "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1", + "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8" ], "index": "pypi", - "version": "==5.4.2" + "version": "==5.4.3" }, "regex": { "hashes": [ - "sha256:1386e75c9d1574f6aa2e4eb5355374c8e55f9aac97e224a8a5a6abded0f9c927", - "sha256:27ff7325b297fb6e5ebb70d10437592433601c423f5acf86e5bc1ee2919b9561", - "sha256:329ba35d711e3428db6b45a53b1b13a0a8ba07cbbcf10bbed291a7da45f106c3", - "sha256:3a9394197664e35566242686d84dfd264c07b20f93514e2e09d3c2b3ffdf78fe", - "sha256:51f17abbe973c7673a61863516bdc9c0ef467407a940f39501e786a07406699c", - "sha256:579ea215c81d18da550b62ff97ee187b99f1b135fd894a13451e00986a080cad", - "sha256:70c14743320a68c5dac7fc5a0f685be63bc2024b062fe2aaccc4acc3d01b14a1", - "sha256:7e61be8a2900897803c293247ef87366d5df86bf701083b6c43119c7c6c99108", - "sha256:8044d1c085d49673aadb3d7dc20ef5cb5b030c7a4fa253a593dda2eab3059929", - "sha256:89d76ce33d3266173f5be80bd4efcbd5196cafc34100fdab814f9b228dee0fa4", - "sha256:99568f00f7bf820c620f01721485cad230f3fb28f57d8fbf4a7967ec2e446994", - "sha256:a7c37f048ec3920783abab99f8f4036561a174f1314302ccfa4e9ad31cb00eb4", - "sha256:c2062c7d470751b648f1cacc3f54460aebfc261285f14bc6da49c6943bd48bdd", - "sha256:c9bce6e006fbe771a02bda468ec40ffccbf954803b470a0345ad39c603402577", - "sha256:ce367d21f33e23a84fb83a641b3834dd7dd8e9318ad8ff677fbfae5915a239f7", - "sha256:ce450ffbfec93821ab1fea94779a8440e10cf63819be6e176eb1973a6017aff5", - "sha256:ce5cc53aa9fbbf6712e92c7cf268274eaff30f6bd12a0754e8133d85a8fb0f5f", - "sha256:d466967ac8e45244b9dfe302bbe5e3337f8dc4dec8d7d10f5e950d83b140d33a", - "sha256:d881c2e657c51d89f02ae4c21d9adbef76b8325fe4d5cf0e9ad62f850f3a98fd", - "sha256:e565569fc28e3ba3e475ec344d87ed3cd8ba2d575335359749298a0899fe122e", - "sha256:ea55b80eb0d1c3f1d8d784264a6764f931e172480a2f1868f2536444c5f01e01" - ], - "version": "==2020.5.14" + "sha256:08997a37b221a3e27d68ffb601e45abfb0093d39ee770e4257bd2f5115e8cb0a", + "sha256:112e34adf95e45158c597feea65d06a8124898bdeac975c9087fe71b572bd938", + "sha256:1700419d8a18c26ff396b3b06ace315b5f2a6e780dad387e4c48717a12a22c29", + "sha256:2f6f211633ee8d3f7706953e9d3edc7ce63a1d6aad0be5dcee1ece127eea13ae", + "sha256:52e1b4bef02f4040b2fd547357a170fc1146e60ab310cdbdd098db86e929b387", + "sha256:55b4c25cbb3b29f8d5e63aeed27b49fa0f8476b0d4e1b3171d85db891938cc3a", + "sha256:5aaa5928b039ae440d775acea11d01e42ff26e1561c0ffcd3d805750973c6baf", + "sha256:654cb773b2792e50151f0e22be0f2b6e1c3a04c5328ff1d9d59c0398d37ef610", + "sha256:690f858d9a94d903cf5cada62ce069b5d93b313d7d05456dbcd99420856562d9", + "sha256:6ad8663c17db4c5ef438141f99e291c4d4edfeaacc0ce28b5bba2b0bf273d9b5", + "sha256:89cda1a5d3e33ec9e231ece7307afc101b5217523d55ef4dc7fb2abd6de71ba3", + "sha256:92d8a043a4241a710c1cf7593f5577fbb832cf6c3a00ff3fc1ff2052aff5dd89", + "sha256:95fa7726d073c87141f7bbfb04c284901f8328e2d430eeb71b8ffdd5742a5ded", + "sha256:97712e0d0af05febd8ab63d2ef0ab2d0cd9deddf4476f7aa153f76feef4b2754", + "sha256:b2ba0f78b3ef375114856cbdaa30559914d081c416b431f2437f83ce4f8b7f2f", + "sha256:bae83f2a56ab30d5353b47f9b2a33e4aac4de9401fb582b55c42b132a8ac3868", + "sha256:c78e66a922de1c95a208e4ec02e2e5cf0bb83a36ceececc10a72841e53fbf2bd", + "sha256:cf59bbf282b627130f5ba68b7fa3abdb96372b24b66bdf72a4920e8153fc7910", + "sha256:e3cdc9423808f7e1bb9c2e0bdb1c9dc37b0607b30d646ff6faf0d4e41ee8fee3", + "sha256:e9b64e609d37438f7d6e68c2546d2cb8062f3adb27e6336bc129b51be20773ac", + "sha256:fbff901c54c22425a5b809b914a3bfaf4b9570eee0e5ce8186ac71eb2025191c" + ], + "version": "==2020.6.8" }, "six": { "hashes": [ @@ -297,10 +353,10 @@ }, "wcwidth": { "hashes": [ - "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1", - "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1" + "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f", + "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f" ], - "version": "==0.1.9" + "version": "==0.2.4" } } } From 93c90e2f126f297bd6924e8c10f9fbe1269280d5 Mon Sep 17 00:00:00 2001 From: Jess Robertson Date: Tue, 9 Jun 2020 11:06:43 +1200 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=A7=AAFactor=20helper=20functions?= =?UTF-8?q?=20out?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit helpers for getting examples now moved to seperate helper module --- setup.cfg | 1 + tests/conftest.py | 5 ++++- tests/helpers/__init__.py | 0 tests/helpers/examples.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/test_examples.py | 31 +++++-------------------------- 5 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/examples.py diff --git a/setup.cfg b/setup.cfg index 689c804..bd6fe2a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,6 +10,7 @@ extend-ignore = console_output_stype = progress log_level = INFO junit_family = legacy +norecursedirs=tests/helpers addopts = --junitxml='tests/reports/test-output.junit.xml' # --cov=app --cov-report term --cov-report xml --cov-report html diff --git a/tests/conftest.py b/tests/conftest.py index 84454d6..f9892a2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,15 @@ from pathlib import Path +import sys import pytest import jsonref as json from fastjsonschema import compile as compile_schema - # Get the root directory of the project ROOT = Path(__file__).parent.parent.absolute() +TEST_DIR = ROOT / "tests" +HELPERS = ROOT / "tests" / "helpers" +sys.path.append(str(HELPERS)) @pytest.fixture() diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/helpers/examples.py b/tests/helpers/examples.py new file mode 100644 index 0000000..1ff87ca --- /dev/null +++ b/tests/helpers/examples.py @@ -0,0 +1,39 @@ +from pathlib import Path +import json +from collections import defaultdict + +# Example files from examples folder +EXAMPLES = Path(__file__).parent.parent.parent / "examples" + + +def filenames(extension=None): + """ + Returns all the example files from the examples directory + + Example files have both a JSON and YAML extension + """ + files = defaultdict(list) + for fname in EXAMPLES.iterdir(): + files[fname.stem].append(fname.suffix) + names = [name for name, ext in files.items() if set(ext) == {".yaml", ".json"}] + + # Wrap in extension if required + if extension is not None: + return [EXAMPLES / f"{name}.{extension}" for name in names] + else: + return names + + +def json_examples(): + "Get all the JSON-formatted example data" + docs = {} + for name, jfile in zip(filenames(), filenames("json")): + with open(jfile, "rb") as source: + data = json.load(source) + + if isinstance(data, list): + for idx, document in enumerate(data): + docs[f"{name}_doc_{id}"] = document + else: + docs[name] = data + return docs diff --git a/tests/test_examples.py b/tests/test_examples.py index 7d466b2..afd4356 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,5 +1,3 @@ -from pathlib import Path -from collections import defaultdict import json from datetime import date, datetime @@ -12,29 +10,10 @@ except ImportError: from yaml import Loader -# Example files from examples folder -EXAMPLES = Path(__file__).parent.parent / "examples" - -# The example files are those for which we have both a JSON and YAML file -FILES = defaultdict(list) -for fname in EXAMPLES.iterdir(): - FILES[fname.stem].append(fname.suffix) -FILENAMES = [name for name, ext in FILES.items() if set(ext) == {".yaml", ".json"}] -YAML_FILES = [f"{fname}.yaml" for fname in FILENAMES] -JSON_FILES = [f"{fname}.json" for fname in FILENAMES] +from helpers.examples import json_examples, filenames # Unwrap all our JSON files into seperate documents -JSON_DOCUMENTS = {} -for jfile in JSON_FILES: - name = jfile.split(".")[0] - with open(EXAMPLES / jfile, "rb") as source: - data = json.load(source) - - if isinstance(data, list): - for idx, document in enumerate(data): - JSON_DOCUMENTS[f"{name}_doc_{idx}"] = document - else: - JSON_DOCUMENTS[name] = data +JSON_DOCUMENTS = json_examples() @pytest.mark.parametrize( @@ -67,12 +46,12 @@ def datetime_handler(obj): @pytest.mark.parametrize( - "yaml_file, json_file", zip(YAML_FILES, JSON_FILES), ids=FILENAMES + "yaml_file, json_file", zip(filenames("yaml"), filenames("json")), ids=filenames() ) def test_yaml_json_equal(yaml_file, json_file): "The YAML and JSON versions should be the same..." # Sanity check yaml data - yaml_file = EXAMPLES / yaml_file + yaml_file = yaml_file assert yaml_file.exists() with open(yaml_file, "r") as source: documents = list(yaml.load_all(source, Loader=Loader)) @@ -81,7 +60,7 @@ def test_yaml_json_equal(yaml_file, json_file): ) # Load and serialize JSON to remove pretty-printing - json_file = EXAMPLES / json_file + json_file = json_file assert json_file.exists() with open(json_file, "rb") as source: data = json.load(source)