From a4079ff15dc2cc0c5ca940967992de3e5e33cac1 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Thu, 29 Aug 2024 08:20:55 +0100 Subject: [PATCH 1/2] Fix #42 Bug with Install Instructions / Requirements --- fuji_server/data/software_file.json | 3 ++- fuji_server/evaluators/fair_evaluator_requirements.py | 10 +++++++--- fuji_server/yaml/metrics_v0.7_software.yaml | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/fuji_server/data/software_file.json b/fuji_server/data/software_file.json index 4859fef5..43674543 100644 --- a/fuji_server/data/software_file.json +++ b/fuji_server/data/software_file.json @@ -24,7 +24,8 @@ ], "parse": "file_name", "pattern": [ - "requirements\\.txt" + "requirements\\.txt", + "pyproject\\.toml" ] }, "docs_directory": { diff --git a/fuji_server/evaluators/fair_evaluator_requirements.py b/fuji_server/evaluators/fair_evaluator_requirements.py index ef70e0ad..3a5d92a7 100644 --- a/fuji_server/evaluators/fair_evaluator_requirements.py +++ b/fuji_server/evaluators/fair_evaluator_requirements.py @@ -58,7 +58,7 @@ def nestedDataContainsKeyword(self, data, key): ) for d in values: if isinstance(d, bytes): - d = d.decode("utf-8") + d = d.decode("utf-8", errors="replace") if isinstance(d, str): if key in d.lower(): return True @@ -88,13 +88,17 @@ def scanForKeywords(self, keywords, locations): content = self.fuji.github_data.get(location) if content is not None: if isinstance(content, bytes): - content = content.decode("utf-8") + content = content.decode("utf-8", errors="replace") if isinstance(content, str): if k in content.lower(): hit_dict[k] = True # found keyword in location - keys_to_check.remove(k) # stop looking, have found something for this key else: hit_dict[k] = self.nestedDataContainsKeyword(content, k) + if hit_dict[k] is True: + keys_to_check.remove(k) # Need to also stop looking + self.logger.info( + f"{self.metric_identifier} : Found {k} in {location}" + ) # Info message for specific keys return hit_dict def testInstructions(self): diff --git a/fuji_server/yaml/metrics_v0.7_software.yaml b/fuji_server/yaml/metrics_v0.7_software.yaml index 3e72fc04..55c7ae9e 100644 --- a/fuji_server/yaml/metrics_v0.7_software.yaml +++ b/fuji_server/yaml/metrics_v0.7_software.yaml @@ -356,6 +356,7 @@ metrics: required: dependency_file: - requirements.txt + - pyproject.toml - target: https://f-uji.net/vocab/metadata/standards modality: all required: @@ -367,7 +368,7 @@ metrics: - deploy created_by: FAIR4RS date_created: 2024-01-18 - date_updated: 2024-01-18 + date_updated: 2024-08-28 version: 0.1 total_score: 2 - metric_identifier: FRSM-14-R1 From 7e5e0309ca795f6fd3c0cbfb6dceb1f97bdca553 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Fri, 30 Aug 2024 08:27:30 +0100 Subject: [PATCH 2/2] Update tests, add full stop to message --- fuji_server/data/repodois.yaml | 19 +++++-- .../evaluators/fair_evaluator_requirements.py | 2 +- .../test_evaluation_software.yaml | 4 +- tests/functional/test_evaluation.py | 54 +++++++++---------- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/fuji_server/data/repodois.yaml b/fuji_server/data/repodois.yaml index db9707e2..713d232d 100644 --- a/fuji_server/data/repodois.yaml +++ b/fuji_server/data/repodois.yaml @@ -4,6 +4,7 @@ ardcx.curtin: https://doi.org/10.17616/R3WM02 ardcx.griffith: https://doi.org/10.17616/R3FG92 ardcx.usq: https://doi.org/10.17616/R36Q18 arfm.ufzdrp: https://doi.org/10.17616/R31NJN1M +awsod.roda: https://doi.org/10.17616/R3M91V awzy.fednnv: https://doi.org/10.17616/R3PM1K bamf.bamf: https://doi.org/10.17616/R31NJMZ5 bf.discover: https://doi.org/10.17616/R31NJMN0 @@ -48,6 +49,7 @@ bl.ukda: https://doi.org/10.17616/R3088K bmbf.hirzqt: https://doi.org/10.17616/R3CK9G brown.bdr: https://doi.org/10.17616/R3193B brvz.rdr: https://doi.org/10.17616/R31NJN5S +bvbw.dhbqgy: https://doi.org/10.17616/R3H35C caltech.data: https://doi.org/10.17616/R3SW99 carl.frdr: https://doi.org/10.17616/R3X50W cbg.datasets: https://doi.org/10.17616/R3PP7J @@ -90,6 +92,7 @@ dkrz.wdcc: https://doi.org/10.17616/R3989R dryad.dryad: https://doi.org/10.17616/R34S33 dzhw.fdz-dzhw: https://doi.org/10.17616/R3BR0G elsevier.md: https://doi.org/10.17616/R3DD11 +epua.rdcrki: https://doi.org/10.17616/R3K31G esdis.ornl: https://doi.org/10.17616/R3SG61 estdoi.keel: https://doi.org/10.17616/R3P636 estdoi.qdb: https://doi.org/10.17616/R31NJMVF @@ -131,8 +134,8 @@ gesis.dezim: https://doi.org/10.17616/R31NJMNK gesis.gesis: https://doi.org/10.17616/R3BB70 gesis.icpsr: https://doi.org/10.17616/R3BC8Q gesis.indepth: https://doi.org/10.17616/R3432T +gesis.iqb: https://doi.org/10.17616/R3M30D gesis.neps: https://doi.org/10.17616/R35P7W -gesis.rki: https://doi.org/10.17616/R3K31G gesis.share: https://doi.org/10.17616/R3BK5N gesis.sodanet: https://doi.org/10.17616/R38G7R gesis.srda: https://doi.org/10.17616/R31NJMLA @@ -141,6 +144,7 @@ gesis.zpid: https://doi.org/10.17616/R3FS48 hain.njltcl: https://doi.org/10.17616/R31NJNAM heliophy.spdf: https://doi.org/10.17616/R3P301 hlqc.znxeli: https://doi.org/10.17616/R31NJMJP +hyfx.gqlzxo: https://doi.org/10.17616/R31NJNDH ieee.dataport: https://doi.org/10.17616/R3H77H iiasa.dare: https://doi.org/10.17616/R31NJMH0 illinois.databank: https://doi.org/10.17616/R3C93F @@ -150,7 +154,7 @@ inist.humanum: https://doi.org/10.17616/R3JM1B inist.ifremer: https://doi.org/10.17616/R31NJMHX inist.ifsttar: https://doi.org/10.17616/R31NJMKM inist.ill: https://doi.org/10.17616/R33H18 -inist.inra: https://doi.org/10.17616/R3DJ4J +inist.osuna: https://doi.org/10.17616/R31NJNKA inist.otelo: https://doi.org/10.17616/R3F19K inist.resif: https://doi.org/10.17616/R37Q06 iris.iris: https://doi.org/10.17616/R3X607 @@ -161,6 +165,7 @@ jbru.bbees: https://doi.org/10.17616/R31NJNEN jcvi.eivbwb: https://doi.org/10.17616/R30P93 jcvi.gxpwaq: https://doi.org/10.17616/R3S634 kaggle.kaggle: https://doi.org/10.17616/R32N5V +kim.colabfit: https://doi.org/10.17616/R31NJNJL kim.openkim: https://doi.org/10.17616/R3SK8T kmot.rdr: https://doi.org/10.17616/R31NJMN6 ktsw.aezvvv: https://doi.org/10.17616/R31NJMVX @@ -173,6 +178,7 @@ mdw.repository: https://doi.org/10.17616/R30M00 mit.physio: https://doi.org/10.17616/R3D06S mlbs.skuxgs: https://doi.org/10.17616/R3ZP8D nasapds.nasapds: https://doi.org/10.17616/R37593 +nbqj.daks: https://doi.org/10.17616/R31NJMZS nkn.nkn: https://doi.org/10.17616/R3JK91 nlqq.xoncsf: https://doi.org/10.17616/R31NJMYB noaa.ncei: https://doi.org/10.17616/R3362J @@ -192,8 +198,13 @@ psu.datacom: https://doi.org/10.17616/R31NJMGV psu.scholar: https://doi.org/10.17616/R34W4H pu.dataspace: https://doi.org/10.17616/R33B6F purdue.purduelib: https://doi.org/10.17616/R3V90N +rdg.prod: https://doi.org/10.17616/R31NJN8R +repod.dbuw: https://doi.org/10.17616/R31NJNHR +repod.mxrdr: https://doi.org/10.17616/R31NJMYU +repod.rds: https://doi.org/10.17616/R31NJMYP rg.rg: https://doi.org/10.17616/R36H16 rice.kinder: https://doi.org/10.17616/R3GB90 +rpak.zvrzin: https://doi.org/10.17616/R31NJNLV sagebio.synapse: https://doi.org/10.17616/R3B934 si.cda: https://doi.org/10.17616/R3201S si.si: https://doi.org/10.17616/R3W49N @@ -224,9 +235,11 @@ tib.kit4radar: https://doi.org/10.17616/R31NJMVU tib.ldeo: https://doi.org/10.17616/R3PV10 tib.luis: https://doi.org/10.17616/R3C519 tib.radar: https://doi.org/10.17616/R3ZX96 +tib.repod: https://doi.org/10.17616/R3JS93 tib.tuhh: https://doi.org/10.17616/R31NJML0 tib.ub-hro: https://doi.org/10.17616/R31NJMQH tib.ubbs: https://doi.org/10.17616/R31NJMTU +tib.ubmr: https://doi.org/10.17616/R31NJMTD tib.ukon: https://doi.org/10.17616/R3M024 tib.zih: https://doi.org/10.17616/R31NJMHY tuw.tethys: https://doi.org/10.17616/R31NJMU0 @@ -237,7 +250,7 @@ umn.drum: https://doi.org/10.17616/R30D19 umn.ipums: https://doi.org/10.17616/R3ZS59 undr.undr: https://doi.org/10.17616/R31NJN9K unimelb.repo1: https://doi.org/10.17616/R3HH10 -unm.dataone: https://doi.org/10.17616/R3101G +unisalzb.anc: https://doi.org/10.17616/R31NJNKJ usda.usda: https://doi.org/10.17616/R3G051 usgs.prod: https://doi.org/10.17616/R33S3S uva.libra: https://doi.org/10.17616/R3TS86 diff --git a/fuji_server/evaluators/fair_evaluator_requirements.py b/fuji_server/evaluators/fair_evaluator_requirements.py index 3a5d92a7..183053ec 100644 --- a/fuji_server/evaluators/fair_evaluator_requirements.py +++ b/fuji_server/evaluators/fair_evaluator_requirements.py @@ -97,7 +97,7 @@ def scanForKeywords(self, keywords, locations): if hit_dict[k] is True: keys_to_check.remove(k) # Need to also stop looking self.logger.info( - f"{self.metric_identifier} : Found {k} in {location}" + f"{self.metric_identifier} : Found {k} in {location}." ) # Info message for specific keys return hit_dict diff --git a/tests/functional/cassettes/test_evaluation/test_evaluation_software.yaml b/tests/functional/cassettes/test_evaluation/test_evaluation_software.yaml index 9450c282..51a2261b 100644 --- a/tests/functional/cassettes/test_evaluation/test_evaluation_software.yaml +++ b/tests/functional/cassettes/test_evaluation/test_evaluation_software.yaml @@ -4549,10 +4549,10 @@ interactions: method: POST uri: http://localhost:1071/fuji/api/v1/evaluate response: - content: "{\"test_id\": \"62b0753c8f4aa9c7c6dc7c433cdcbfb330edbd92\", \"request\": {\"object_identifier\": \"https://github.com/pangaea-data-publisher/fuji\", \"test_debug\": true, \"use_datacite\": true, \"use_github\": true, \"metric_version\": \"metrics_v0.7_software\"}, \"resolved_url\": \"https://github.com/pangaea-data-publisher/fuji\", \"start_timestamp\": \"2024-05-03T14:42:02Z\", \"end_timestamp\": \"2024-05-03T14:42:33Z\", \"metric_specification\": \"https://doi.org/10.5281/zenodo.6461229\", \"metric_version\": \"metrics_v0.7_software\", \"software_version\": \"3.2.0\", \"total_metrics\": 17, \"summary\": {\"score_earned\": {\"A\": 0, \"F\": 0, \"I\": 0, \"R\": 4, \"A1\": 0, \"F1\": 0, \"F1.1\": 0, \"F1.2\": 0, \"F2\": 0, \"F3\": 0, \"F4\": 0, \"I1\": 0, \"I2\": 0, \"R1\": 0, \"R1.1\": 4, \"R1.2\": 0, \"FAIR\": 4.0}, \"score_total\": {\"A\": 2, \"F\": 20, \"I\": 7, \"R\": 16, \"A1\": 2, \"F1\": 3, \"F1.1\": 3, \"F1.2\": 3, \"F2\": 6, \"F3\": 2, \"F4\": 3, \"I1\": 6, \"I2\": 1, \"R1\": 8, \"R1.1\": 5, \"R1.2\": 3, \"FAIR\": 45.0}, \"score_percent\": {\"A\": 0.0, \"F\": 0.0, \"I\": 0.0, \"R\": 25.0, \"A1\": 0.0, \"F1\": 0.0, \"F1.1\": 0.0, \"F1.2\": 0.0, \"F2\": 0.0, \"F3\": 0.0, \"F4\": 0.0, \"I1\": 0.0, \"I2\": 0.0, \"R1\": 0.0, \"R1.1\": 80.0, \"R1.2\": 0.0, \"FAIR\": 8.89}, \"status_total\": {\"A1\": 1, \"F1\": 1, \"F1.1\": 1, \"F1.2\": 1, \"F2\": 2, \"F3\": 1, \"F4\": 1, \"I1\": 2, \"I2\": 1, \"R1\": 3, \"R1.1\": 2, \"R1.2\": 1, \"A\": 1, \"F\": 7, \"I\": 3, \"R\": 6, \"FAIR\": 17}, \"status_passed\": {\"A1\": 0, \"F1\": 0, \"F1.1\": 0, \"F1.2\": 0, \"F2\": 0, \"F3\": 0, \"F4\": 0, \"I1\": 0, \"I2\": 0, \"R1\": 0, \"R1.1\": 2, \"R1.2\": 0, \"A\": 0, \"F\": 0, \"I\": 0, \"R\": 2, \"FAIR\": 2}, \"maturity\": {\"A\": 0, \"F\": 0, \"I\": 0, \"R\": 1, \"A1\": 0, \"F1\": 0, \"F1.1\": 0, \"F1.2\": 0, \"F2\": 0, \"F3\": 0, \"F4\": 0, \"I1\": 0, \"I2\": 0, \"R1\": 0, \"R1.1\": 3, \"R1.2\": 0, \"FAIR\": 1.0}}, \"results\": [{\"id\": 1, \"metric_identifier\": \"FRSM-01-F1\", \"metric_name\": \"Does the software have a globally unique and persistent identifier?\", \"metric_tests\": {\"FRSM-01-F1-1\": {\"metric_test_name\": \"The software has a human and machine-readable unique identifier that is resolvable to a machine-readable landing page and follows a defined unique identifier syntax.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-01-F1-2\": {\"metric_test_name\": \"The identifier uses an identifier scheme that guarantees globally uniqueness and persistence.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-01-F1-3\": {\"metric_test_name\": \"The identifier scheme is commonly used in the domain.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for unique identifier is not implemented.\", \"WARNING: Test for identifier scheme is not implemented.\", \"WARNING: Test for domain use of identifier scheme is not implemented.\", \"WARNING: Failed to check the software identifier.\"]}, {\"id\": 2, \"metric_identifier\": \"FRSM-02-F1.1\", \"metric_name\": \"Do the different components of the software have their own identifiers?\", \"metric_tests\": {\"FRSM-02-F1.1-1\": {\"metric_test_name\": \"Where the 'software' consists of multiple distinct components, each component has a distinct identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-02-F1.1-2\": {\"metric_test_name\": \"The relationship between components is embodied in the identifier metadata.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-02-F1.1-3\": {\"metric_test_name\": \"Every component to granularity level GL3 (module) has its own unique identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for distinct identifiers is not implemented.\", \"WARNING: Test for identifier metadata is not implemented.\", \"WARNING: Test for GL3 (module) identifiers is not implemented.\", \"WARNING: Failed to check the software component identifiers.\"]}, {\"id\": 3, \"metric_identifier\": \"FRSM-03-F1.2\", \"metric_name\": \"Does each version of the software have a unique identifier?\", \"metric_tests\": {\"FRSM-03-F1.2-1\": {\"metric_test_name\": \"Each version of the software has a different identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-03-F1.2-2\": {\"metric_test_name\": \"Relations between the versions are included in the identifier metadata.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-03-F1.2-3\": {\"metric_test_name\": \"The version number is included in the identifier metadata.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for distinct identifiers is not implemented.\", \"WARNING: Test for identifier metadata is not implemented.\", \"WARNING: Test for version number in identifier is not implemented.\", \"WARNING: Failed to check the software version identifier.\"]}, {\"id\": 4, \"metric_identifier\": \"FRSM-04-F2\", \"metric_name\": \"Does the software include descriptive metadata which helps define its purpose?\", \"metric_tests\": {\"FRSM-04-F2-1\": {\"metric_test_name\": \"The software includes a README or other file which includes the software title and description.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-04-F2-2\": {\"metric_test_name\": \"The software includes other descriptive metadata such as domain, funder, programming language, date created, and keywords.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-04-F2-3\": {\"metric_test_name\": \"The metadata is contained in a format such as CodeMeta or ProjectObjectModel that enables full machine actionability.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {\"core_metadata_status\": \"insufficient metadata\", \"core_metadata_found\": {\"summary\": \"FAIRsFAIR Research Data Object Assessment Service. Contribute to pangaea-data-publisher/fuji development by creating an account on GitHub.\"}, \"core_metadata_source\": [[\"MICRODATA_EMBEDDED\", \"microdata_rdfa\"], [\"OPENGRAPH_EMBEDDED\", \"html_embedding\"], [\"DUBLINCORE_EMBEDDED\", \"html_embedding\"]]}, \"test_debug\": [\"INFO: Testing if any metadata has been made available via common web standards\", \"WARNING: Test for descriptive metadata is not implemented for FRSM.\", \"WARNING: Not all required core descriptive metadata elements exist, missing -: ['publication_date', 'publisher', 'keywords', 'creator', 'object_type', 'object_identifier', 'title']\", \"WARNING: Test for minimum metadata is not implemented.\", \"WARNING: Test for metadata format is not implemented.\"]}, {\"id\": 5, \"metric_identifier\": \"FRSM-05-R1\", \"metric_name\": \"Does the software include development metadata which helps define its status?\", \"metric_tests\": {\"FRSM-05-R1-1\": {\"metric_test_name\": \"The software includes metadata for contact or support in the README or other intrinsic metadata file according to community standards.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-05-R1-2\": {\"metric_test_name\": \"The software includes metadata for development status, links to documentation.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-05-R1-3\": {\"metric_test_name\": \"The metadata is contained in a format such as CodeMeta or ProjectObjectModel that enables full machine-actionability.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for contact information is not implemented.\", \"WARNING: Test for development status in metadata is not implemented.\", \"WARNING: Test for metadata format is not implemented.\", \"WARNING: Failed to check the software development metadata.\"]}, {\"id\": 6, \"metric_identifier\": \"FRSM-06-F2\", \"metric_name\": \"Does the software include metadata about the contributors and their roles?\", \"metric_tests\": {\"FRSM-06-F2-1\": {\"metric_test_name\": \"The software includes metadata about the contributors.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-06-F2-2\": {\"metric_test_name\": \"The software includes citation metadata that includes all contributors and their roles. This includes ORCIDs when contributors have them.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-06-F2-3\": {\"metric_test_name\": \"Does the citation metadata include the proportional credit attributed to each contributor?\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {\"provenance_metadata_included\": {\"is_available\": false, \"provenance_metadata\": []}, \"structured_provenance_available\": null}, \"test_debug\": [\"WARNING: Test for descriptive metadata is not implemented for FRSM.\", \"INFO: Check if provenance information is available in descriptive metadata\", \"INFO: Check if provenance information is available in metadata about related resources\", \"WARNING: No provenance information found in metadata about related resources\", \"WARNING: Test for citation metadata is not implemented.\", \"WARNING: Test for proportional credit in citation metadata is not implemented.\"]}, {\"id\": 7, \"metric_identifier\": \"FRSM-07-F3\", \"metric_name\": \"Does the software metadata include the identifier for the software?\", \"metric_tests\": {\"FRSM-07-F3-1\": {\"metric_test_name\": \"Does the software include an identifier in the README or citation file?\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-07-F3-2\": {\"metric_test_name\": \"Does the identifier resolve to the same instance of the software?\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 2}, \"maturity\": 0, \"output\": {\"object_identifier_included\": null, \"object_content_identifier_included\": []}, \"test_debug\": [\"WARNING: Test for identifier resolve target is not implemented.\", \"WARNING: Valid data (content) identifier missing.\"]}, {\"id\": 8, \"metric_identifier\": \"FRSM-08-F4\", \"metric_name\": \"Does the software have a publicly available, openly accessible and persistent metadata record?\", \"metric_tests\": {\"FRSM-08-F4-1\": {\"metric_test_name\": \"A metadata record for the software is present on an infrastructure that guarantees persistence.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-08-F4-2\": {\"metric_test_name\": \"The persistent metadata record is available through public search engines. The metadata has a globally unique and persistent identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-08-F4-3\": {\"metric_test_name\": \"The persistent metadata record is available through multiple, cross-referenced infrastructures.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": [], \"test_debug\": [\"WARNING: Test for descriptive metadata is not implemented for FRSM.\", \"WARNING: Test for availability through public search engine is not implemented.\", \"WARNING: Test for availability through multiple, cross-referenced infrastructures is not implemented.\"]}, {\"id\": 9, \"metric_identifier\": \"FRSM-09-A1\", \"metric_name\": \"Is the software developed in a code repository / forge that uses standard communications protocols?\", \"metric_tests\": {\"FRSM-09-A1-1\": {\"metric_test_name\": \"The code repository / forge can be accessed using the identifier via a standardised protocol.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-09-A1-2\": {\"metric_test_name\": \"If authentication or authorisation are required, these are supported by the communication protocols and the repository / forge.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 2}, \"maturity\": 0, \"output\": {\"standard_data_protocol\": {}}, \"test_debug\": [\"WARNING: Test for standard protocol is not implemented for FRSM.\", \"WARNING: Skipping protocol test for data since NO content (data) identifier is given in metadata\"]}, {\"id\": 10, \"metric_identifier\": \"FRSM-10-I1\", \"metric_name\": \"Are the formats used by the data consumed or produced by the software open and a reference provided to the format?\", \"metric_tests\": {\"FRSM-10-I1-1\": {\"metric_test_name\": \"The documentation describes the data formats used.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-10-I1-2\": {\"metric_test_name\": \"The data formats used are open.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-10-I1-3\": {\"metric_test_name\": \"A reference to the schema is provided.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": [], \"test_debug\": [\"WARNING: Test for documentation abouut data formats is not implemented.\", \"WARNING: Test for openness of data formats is not implemented.\", \"WARNING: Test for presence of schema reference is not implemented.\", \"WARNING: Could not perform file format checks as data content identifier(s) unavailable/inaccesible\"]}, {\"id\": 11, \"metric_identifier\": \"FRSM-11-I1\", \"metric_name\": \"Does the software use open APIs that support machine-readable interface definition?\", \"metric_tests\": {\"FRSM-11-I1-1\": {\"metric_test_name\": \"The software provides documented APIs.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-11-I1-2\": {\"metric_test_name\": \"The APIs are open (freely accessible).\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-11-I1-3\": {\"metric_test_name\": \"The APIs include a machine-readable interface definition.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for documented API is not implemented.\", \"WARNING: Test for openness of API is not implemented.\", \"WARNING: Test for machine-readable interface definition is not implemented.\", \"WARNING: Failed to check the software API.\"]}, {\"id\": 12, \"metric_identifier\": \"FRSM-12-I2\", \"metric_name\": \"Does the software provide references to other objects that support its use?\", \"metric_tests\": {\"FRSM-12-I2-1\": {\"metric_test_name\": \"The software metadata includes machine-readable references to articles describing the software, articles demonstrating use of the software, or to the data it uses.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 1}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"INFO: No debug messages received\"]}, {\"id\": 13, \"metric_identifier\": \"FRSM-13-R1\", \"metric_name\": \"Does the software describe what is required to use it?\", \"metric_tests\": {\"FRSM-13-R1-1\": {\"metric_test_name\": \"The software has build, installation and/or execution instructions.\", \"metric_test_requirements\": [{\"modality\": \"any\", \"required\": {\"location\": [\"README\", \"docs_directory\", \"wiki\"], \"keywords\": [\"build\", \"install\", \"run\"]}, \"tested_on\": null, \"comment\": null, \"target\": \"https://f-uji.net/vocab/metadata/standards\"}], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-13-R1-2\": {\"metric_test_name\": \"Dependencies are provided in a machine-readable format and the building and installation of the software is automated.\", \"metric_test_requirements\": [{\"modality\": \"any\", \"required\": {\"dependency_file\": [\"requirements.txt\"]}, \"tested_on\": null, \"comment\": null, \"target\": \"https://f-uji.net/vocab/metadata/standards\"}, {\"modality\": \"all\", \"required\": {\"automation_file\": [\"Jenkinsfile\", \"github_actions\"], \"automation_keywords\": [\"build\", \"deploy\"]}, \"tested_on\": null, \"comment\": null, \"target\": \"https://f-uji.net/vocab/metadata/standards\"}], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 2}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"INFO: Looking for any keywords ['build', 'install', 'run'] in ['README', 'docs_directory', 'wiki'] (FRSM-13-R1-1).\", \"WARNING: Did not find any keywords ['build', 'install', 'run'] in ['README', 'docs_directory', 'wiki'] (FRSM-13-R1-1).\", \"INFO: Checking presence of any of ['requirements.txt'] (FRSM-13-R1-2).\", \"INFO: Looking for all keywords ['build', 'deploy'] in ['Jenkinsfile', 'github_actions'] (FRSM-13-R1-2).\", \"WARNING: Did not find any of ['requirements.txt'] (FRSM-13-R1-2).\", \"WARNING: Did not find all keywords ['build', 'deploy'] in ['Jenkinsfile', 'github_actions'] (FRSM-13-R1-2).\", \"WARNING: Failed to check the software requirements.\"]}, {\"id\": 14, \"metric_identifier\": \"FRSM-14-R1\", \"metric_name\": \"Does the software come with test cases to demonstrate it is working?\", \"metric_tests\": {\"FRSM-14-R1-1\": {\"metric_test_name\": \"Tests and data are provided to check that the software is operating as expected.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-14-R1-2\": {\"metric_test_name\": \"Automated unit and system tests are provided.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-14-R1-3\": {\"metric_test_name\": \"Code coverage / test coverage is reported.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for presence of tests and test data is not implemented.\", \"WARNING: Test for Automated unit and system tests is not implemented.\", \"WARNING: Test for code coverage is not implemented.\", \"WARNING: Failed to check the software version identifier.\"]}, {\"id\": 15, \"metric_identifier\": \"FRSM-15-R1.1\", \"metric_name\": \"The software source code includes licensing information for the software and any bundled external software.\", \"metric_tests\": {\"FRSM-15-R1.1-1\": {\"metric_test_name\": \"License file is included.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 1, \"metric_test_status\": \"pass\"}, \"FRSM-15-R1.1-2\": {\"metric_test_name\": \"The source code includes licensing information for all components bundled with that software.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-15-R1.1-3\": {\"metric_test_name\": \"Recognized licence is in SPDX format.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 3, \"metric_test_status\": \"pass\"}}, \"test_status\": \"pass\", \"score\": {\"earned\": 2, \"total\": 3}, \"maturity\": 3, \"output\": [{\"license\": \"MIT License\", \"osi_approved\": true, \"details_url\": \"http://spdx.org/licenses/MIT.html\"}], \"test_debug\": [\"INFO: License verification name through SPDX registry -: MIT License\", \"INFO: Found SPDX license representation -: http://spdx.org/licenses/MIT.json\", \"SUCCESS: Found SPDX license representation (spdx url, osi_approved)\", \"SUCCESS: Found licence file: ['LICENSE', 'fuji_server/models/license.py'].\", \"INFO: Will consider all SPDX licenses as community specific licenses for FRSM-15-R1.1\", \"WARNING: Test for license information of bundled components is not implemented (FRSM-15-R1.1-2).\"]}, {\"id\": 16, \"metric_identifier\": \"FRSM-16-R1.1\", \"metric_name\": \"Does the software metadata record include licensing information?\", \"metric_tests\": {\"FRSM-16-R1.1-1\": {\"metric_test_name\": \"The identifier or metadata record includes licensing and copyright information.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 1, \"metric_test_status\": \"pass\"}, \"FRSM-16-R1.1-2\": {\"metric_test_name\": \"The software licensing information is in SPDX format, or other machine-readable form.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 3, \"metric_test_status\": \"pass\"}}, \"test_status\": \"pass\", \"score\": {\"earned\": 2, \"total\": 2}, \"maturity\": 3, \"output\": [{\"license\": \"MIT License\", \"osi_approved\": true, \"details_url\": \"http://spdx.org/licenses/MIT.html\"}], \"test_debug\": [\"INFO: License verification name through SPDX registry -: MIT License\", \"INFO: Found SPDX license representation -: http://spdx.org/licenses/MIT.json\", \"SUCCESS: Found SPDX license representation (spdx url, osi_approved)\", \"SUCCESS: Found licence information in metadata\", \"INFO: Will consider all SPDX licenses as community specific licenses for FRSM-16-R1.1\"]}, {\"id\": 17, \"metric_identifier\": \"FRSM-17-R1.2\", \"metric_name\": \"Does the software include provenance information that describe the development of the software?\", \"metric_tests\": {\"FRSM-17-R1.2-1\": {\"metric_test_name\": \"The software source code repository / forge includes a commit history.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-17-R1.2-2\": {\"metric_test_name\": \"The software source code repository links commits to issues / tickets.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-17-R1.2-3\": {\"metric_test_name\": \"The software project uses other tools to capture detailed machine readable provenance information.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for commit history is not implemented.\", \"WARNING: Test for commit linkage to issues is not implemented.\", \"WARNING: Test for other tools capturing provenance information is not implemented.\"]}]}\n" + content: "{\"test_id\": \"62b0753c8f4aa9c7c6dc7c433cdcbfb330edbd92\", \"request\": {\"object_identifier\": \"https://github.com/pangaea-data-publisher/fuji\", \"test_debug\": true, \"use_datacite\": true, \"use_github\": true, \"metric_version\": \"metrics_v0.7_software\"}, \"resolved_url\": \"https://github.com/pangaea-data-publisher/fuji\", \"start_timestamp\": \"2024-08-30T06:57:57Z\", \"end_timestamp\": \"2024-08-30T06:58:24Z\", \"metric_specification\": \"https://doi.org/10.5281/zenodo.6461229\", \"metric_version\": \"metrics_v0.7_software\", \"software_version\": \"3.2.0\", \"total_metrics\": 17, \"summary\": {\"score_earned\": {\"A\": 0, \"F\": 0, \"I\": 0, \"R\": 5, \"A1\": 0, \"F1\": 0, \"F1.1\": 0, \"F1.2\": 0, \"F2\": 0, \"F3\": 0, \"F4\": 0, \"I1\": 0, \"I2\": 0, \"R1\": 1, \"R1.1\": 4, \"R1.2\": 0, \"FAIR\": 5}, \"score_total\": {\"A\": 2, \"F\": 20, \"I\": 7, \"R\": 16, \"A1\": 2, \"F1\": 3, \"F1.1\": 3, \"F1.2\": 3, \"F2\": 6, \"F3\": 2, \"F4\": 3, \"I1\": 6, \"I2\": 1, \"R1\": 8, \"R1.1\": 5, \"R1.2\": 3, \"FAIR\": 45}, \"score_percent\": {\"A\": 0, \"F\": 0, \"I\": 0, \"R\": 31.25, \"A1\": 0, \"F1\": 0, \"F1.1\": 0, \"F1.2\": 0, \"F2\": 0, \"F3\": 0, \"F4\": 0, \"I1\": 0, \"I2\": 0, \"R1\": 12.5, \"R1.1\": 80, \"R1.2\": 0, \"FAIR\": 11.11}, \"status_total\": {\"A1\": 1, \"F1\": 1, \"F1.1\": 1, \"F1.2\": 1, \"F2\": 2, \"F3\": 1, \"F4\": 1, \"I1\": 2, \"I2\": 1, \"R1\": 3, \"R1.1\": 2, \"R1.2\": 1, \"A\": 1, \"F\": 7, \"I\": 3, \"R\": 6, \"FAIR\": 17}, \"status_passed\": {\"A1\": 0, \"F1\": 0, \"F1.1\": 0, \"F1.2\": 0, \"F2\": 0, \"F3\": 0, \"F4\": 0, \"I1\": 0, \"I2\": 0, \"R1\": 1, \"R1.1\": 2, \"R1.2\": 0, \"A\": 0, \"F\": 0, \"I\": 0, \"R\": 3, \"FAIR\": 3}, \"maturity\": {\"A\": 0, \"F\": 0, \"I\": 0, \"R\": 1, \"A1\": 0, \"F1\": 0, \"F1.1\": 0, \"F1.2\": 0, \"F2\": 0, \"F3\": 0, \"F4\": 0, \"I1\": 0, \"I2\": 0, \"R1\": 1, \"R1.1\": 3, \"R1.2\": 0, \"FAIR\": 1}}, \"results\": [{\"id\": 1, \"metric_identifier\": \"FRSM-01-F1\", \"metric_name\": \"Does the software have a globally unique and persistent identifier?\", \"metric_tests\": {\"FRSM-01-F1-1\": {\"metric_test_name\": \"The software has a human and machine-readable unique identifier that is resolvable to a machine-readable landing page and follows a defined unique identifier syntax.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-01-F1-2\": {\"metric_test_name\": \"The identifier uses an identifier scheme that guarantees globally uniqueness and persistence.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-01-F1-3\": {\"metric_test_name\": \"The identifier scheme is commonly used in the domain.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for unique identifier is not implemented.\", \"WARNING: Test for identifier scheme is not implemented.\", \"WARNING: Test for domain use of identifier scheme is not implemented.\", \"WARNING: Failed to check the software identifier.\"]}, {\"id\": 2, \"metric_identifier\": \"FRSM-02-F1.1\", \"metric_name\": \"Do the different components of the software have their own identifiers?\", \"metric_tests\": {\"FRSM-02-F1.1-1\": {\"metric_test_name\": \"Where the 'software' consists of multiple distinct components, each component has a distinct identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-02-F1.1-2\": {\"metric_test_name\": \"The relationship between components is embodied in the identifier metadata.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-02-F1.1-3\": {\"metric_test_name\": \"Every component to granularity level GL3 (module) has its own unique identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for distinct identifiers is not implemented.\", \"WARNING: Test for identifier metadata is not implemented.\", \"WARNING: Test for GL3 (module) identifiers is not implemented.\", \"WARNING: Failed to check the software component identifiers.\"]}, {\"id\": 3, \"metric_identifier\": \"FRSM-03-F1.2\", \"metric_name\": \"Does each version of the software have a unique identifier?\", \"metric_tests\": {\"FRSM-03-F1.2-1\": {\"metric_test_name\": \"Each version of the software has a different identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-03-F1.2-2\": {\"metric_test_name\": \"Relations between the versions are included in the identifier metadata.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-03-F1.2-3\": {\"metric_test_name\": \"The version number is included in the identifier metadata.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for distinct identifiers is not implemented.\", \"WARNING: Test for identifier metadata is not implemented.\", \"WARNING: Test for version number in identifier is not implemented.\", \"WARNING: Failed to check the software version identifier.\"]}, {\"id\": 4, \"metric_identifier\": \"FRSM-04-F2\", \"metric_name\": \"Does the software include descriptive metadata which helps define its purpose?\", \"metric_tests\": {\"FRSM-04-F2-1\": {\"metric_test_name\": \"The software includes a README or other file which includes the software title and description.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-04-F2-2\": {\"metric_test_name\": \"The software includes other descriptive metadata such as domain, funder, programming language, date created, and keywords.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-04-F2-3\": {\"metric_test_name\": \"The metadata is contained in a format such as CodeMeta or ProjectObjectModel that enables full machine actionability.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {\"core_metadata_status\": \"insufficient metadata\", \"core_metadata_found\": {\"summary\": \"FAIRsFAIR Research Data Object Assessment Service. Contribute to pangaea-data-publisher/fuji development by creating an account on GitHub.\"}, \"core_metadata_source\": [[\"OPENGRAPH_EMBEDDED\", \"html_embedding\"], [\"DUBLINCORE_EMBEDDED\", \"html_embedding\"], [\"MICRODATA_EMBEDDED\", \"microdata_rdfa\"]]}, \"test_debug\": [\"INFO: Testing if any metadata has been made available via common web standards\", \"WARNING: Test for descriptive metadata is not implemented for FRSM.\", \"WARNING: Not all required core descriptive metadata elements exist, missing -: ['keywords', 'object_type', 'publication_date', 'publisher', 'creator', 'title', 'object_identifier']\", \"WARNING: Test for minimum metadata is not implemented.\", \"WARNING: Test for metadata format is not implemented.\"]}, {\"id\": 5, \"metric_identifier\": \"FRSM-05-R1\", \"metric_name\": \"Does the software include development metadata which helps define its status?\", \"metric_tests\": {\"FRSM-05-R1-1\": {\"metric_test_name\": \"The software includes metadata for contact or support in the README or other intrinsic metadata file according to community standards.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-05-R1-2\": {\"metric_test_name\": \"The software includes metadata for development status, links to documentation.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-05-R1-3\": {\"metric_test_name\": \"The metadata is contained in a format such as CodeMeta or ProjectObjectModel that enables full machine-actionability.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for contact information is not implemented.\", \"WARNING: Test for development status in metadata is not implemented.\", \"WARNING: Test for metadata format is not implemented.\", \"WARNING: Failed to check the software development metadata.\"]}, {\"id\": 6, \"metric_identifier\": \"FRSM-06-F2\", \"metric_name\": \"Does the software include metadata about the contributors and their roles?\", \"metric_tests\": {\"FRSM-06-F2-1\": {\"metric_test_name\": \"The software includes metadata about the contributors.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-06-F2-2\": {\"metric_test_name\": \"The software includes citation metadata that includes all contributors and their roles. This includes ORCIDs when contributors have them.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-06-F2-3\": {\"metric_test_name\": \"Does the citation metadata include the proportional credit attributed to each contributor?\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {\"provenance_metadata_included\": {\"is_available\": false, \"provenance_metadata\": []}, \"structured_provenance_available\": null}, \"test_debug\": [\"WARNING: Test for descriptive metadata is not implemented for FRSM.\", \"INFO: Check if provenance information is available in descriptive metadata\", \"INFO: Check if provenance information is available in metadata about related resources\", \"WARNING: No provenance information found in metadata about related resources\", \"WARNING: Test for citation metadata is not implemented.\", \"WARNING: Test for proportional credit in citation metadata is not implemented.\"]}, {\"id\": 7, \"metric_identifier\": \"FRSM-07-F3\", \"metric_name\": \"Does the software metadata include the identifier for the software?\", \"metric_tests\": {\"FRSM-07-F3-1\": {\"metric_test_name\": \"Does the software include an identifier in the README or citation file?\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-07-F3-2\": {\"metric_test_name\": \"Does the identifier resolve to the same instance of the software?\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 2}, \"maturity\": 0, \"output\": {\"object_identifier_included\": null, \"object_content_identifier_included\": []}, \"test_debug\": [\"WARNING: Test for identifier resolve target is not implemented.\", \"WARNING: Valid data (content) identifier missing.\"]}, {\"id\": 8, \"metric_identifier\": \"FRSM-08-F4\", \"metric_name\": \"Does the software have a publicly available, openly accessible and persistent metadata record?\", \"metric_tests\": {\"FRSM-08-F4-1\": {\"metric_test_name\": \"A metadata record for the software is present on an infrastructure that guarantees persistence.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-08-F4-2\": {\"metric_test_name\": \"The persistent metadata record is available through public search engines. The metadata has a globally unique and persistent identifier.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-08-F4-3\": {\"metric_test_name\": \"The persistent metadata record is available through multiple, cross-referenced infrastructures.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": [], \"test_debug\": [\"WARNING: Test for descriptive metadata is not implemented for FRSM.\", \"WARNING: Test for availability through public search engine is not implemented.\", \"WARNING: Test for availability through multiple, cross-referenced infrastructures is not implemented.\"]}, {\"id\": 9, \"metric_identifier\": \"FRSM-09-A1\", \"metric_name\": \"Is the software developed in a code repository / forge that uses standard communications protocols?\", \"metric_tests\": {\"FRSM-09-A1-1\": {\"metric_test_name\": \"The code repository / forge can be accessed using the identifier via a standardised protocol.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-09-A1-2\": {\"metric_test_name\": \"If authentication or authorisation are required, these are supported by the communication protocols and the repository / forge.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 2}, \"maturity\": 0, \"output\": {\"standard_data_protocol\": {}}, \"test_debug\": [\"WARNING: Test for standard protocol is not implemented for FRSM.\", \"WARNING: Skipping protocol test for data since NO content (data) identifier is given in metadata\"]}, {\"id\": 10, \"metric_identifier\": \"FRSM-10-I1\", \"metric_name\": \"Are the formats used by the data consumed or produced by the software open and a reference provided to the format?\", \"metric_tests\": {\"FRSM-10-I1-1\": {\"metric_test_name\": \"The documentation describes the data formats used.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-10-I1-2\": {\"metric_test_name\": \"The data formats used are open.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-10-I1-3\": {\"metric_test_name\": \"A reference to the schema is provided.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": [], \"test_debug\": [\"WARNING: Test for documentation abouut data formats is not implemented.\", \"WARNING: Test for openness of data formats is not implemented.\", \"WARNING: Test for presence of schema reference is not implemented.\", \"WARNING: Could not perform file format checks as data content identifier(s) unavailable/inaccesible\"]}, {\"id\": 11, \"metric_identifier\": \"FRSM-11-I1\", \"metric_name\": \"Does the software use open APIs that support machine-readable interface definition?\", \"metric_tests\": {\"FRSM-11-I1-1\": {\"metric_test_name\": \"The software provides documented APIs.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-11-I1-2\": {\"metric_test_name\": \"The APIs are open (freely accessible).\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-11-I1-3\": {\"metric_test_name\": \"The APIs include a machine-readable interface definition.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for documented API is not implemented.\", \"WARNING: Test for openness of API is not implemented.\", \"WARNING: Test for machine-readable interface definition is not implemented.\", \"WARNING: Failed to check the software API.\"]}, {\"id\": 12, \"metric_identifier\": \"FRSM-12-I2\", \"metric_name\": \"Does the software provide references to other objects that support its use?\", \"metric_tests\": {\"FRSM-12-I2-1\": {\"metric_test_name\": \"The software metadata includes machine-readable references to articles describing the software, articles demonstrating use of the software, or to the data it uses.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 1}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"INFO: No debug messages received\"]}, {\"id\": 13, \"metric_identifier\": \"FRSM-13-R1\", \"metric_name\": \"Does the software describe what is required to use it?\", \"metric_tests\": {\"FRSM-13-R1-1\": {\"metric_test_name\": \"The software has build, installation and/or execution instructions.\", \"metric_test_requirements\": [{\"modality\": \"any\", \"required\": {\"location\": [\"README\", \"docs_directory\", \"wiki\"], \"keywords\": [\"install\"]}, \"tested_on\": null, \"comment\": null, \"target\": \"https://f-uji.net/vocab/metadata/standards\"}], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 1, \"metric_test_status\": \"pass\"}, \"FRSM-13-R1-2\": {\"metric_test_name\": \"Dependencies are provided in a machine-readable format and the building and installation of the software is automated.\", \"metric_test_requirements\": [{\"modality\": \"any\", \"required\": {\"dependency_file\": [\"requirements.txt\", \"pyproject.toml\"]}, \"tested_on\": null, \"comment\": null, \"target\": \"https://f-uji.net/vocab/metadata/standards\"}, {\"modality\": \"all\", \"required\": {\"automation_file\": [\"Jenkinsfile\", \"github_actions\"], \"automation_keywords\": [\"build\", \"deploy\"]}, \"tested_on\": null, \"comment\": null, \"target\": \"https://f-uji.net/vocab/metadata/standards\"}], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"pass\", \"score\": {\"earned\": 1, \"total\": 2}, \"maturity\": 1, \"output\": {}, \"test_debug\": [\"INFO: Looking for any keywords ['build', 'install', 'run'] in ['README', 'docs_directory', 'wiki'] (FRSM-13-R1-1).\", \"INFO: Found build in README.\", \"INFO: Found run in README.\", \"SUCCESS: Found required keywords (FRSM-13-R1-1).\", \"INFO: Checking presence of any of ['requirements.txt', 'pyproject.toml'] (FRSM-13-R1-2).\", \"INFO: Looking for all keywords ['build', 'deploy'] in ['Jenkinsfile', 'github_actions'] (FRSM-13-R1-2).\", \"INFO: Found dependency files (FRSM-13-R1-2).\", \"WARNING: Did not find all keywords ['build', 'deploy'] in ['Jenkinsfile', 'github_actions'] (FRSM-13-R1-2).\"]}, {\"id\": 14, \"metric_identifier\": \"FRSM-14-R1\", \"metric_name\": \"Does the software come with test cases to demonstrate it is working?\", \"metric_tests\": {\"FRSM-14-R1-1\": {\"metric_test_name\": \"Tests and data are provided to check that the software is operating as expected.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-14-R1-2\": {\"metric_test_name\": \"Automated unit and system tests are provided.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-14-R1-3\": {\"metric_test_name\": \"Code coverage / test coverage is reported.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for presence of tests and test data is not implemented.\", \"WARNING: Test for Automated unit and system tests is not implemented.\", \"WARNING: Test for code coverage is not implemented.\", \"WARNING: Failed to check the software version identifier.\"]}, {\"id\": 15, \"metric_identifier\": \"FRSM-15-R1.1\", \"metric_name\": \"The software source code includes licensing information for the software and any bundled external software.\", \"metric_tests\": {\"FRSM-15-R1.1-1\": {\"metric_test_name\": \"License file is included.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 1, \"metric_test_status\": \"pass\"}, \"FRSM-15-R1.1-2\": {\"metric_test_name\": \"The source code includes licensing information for all components bundled with that software.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-15-R1.1-3\": {\"metric_test_name\": \"Recognized licence is in SPDX format.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 3, \"metric_test_status\": \"pass\"}}, \"test_status\": \"pass\", \"score\": {\"earned\": 2, \"total\": 3}, \"maturity\": 3, \"output\": [{\"license\": \"MIT License\", \"osi_approved\": true, \"details_url\": \"http://spdx.org/licenses/MIT.html\"}], \"test_debug\": [\"INFO: License verification name through SPDX registry -: MIT License\", \"INFO: Found SPDX license representation -: http://spdx.org/licenses/MIT.json\", \"SUCCESS: Found SPDX license representation (spdx url, osi_approved)\", \"SUCCESS: Found licence file: ['LICENSE', 'fuji_server/models/license.py'].\", \"INFO: Will consider all SPDX licenses as community specific licenses for FRSM-15-R1.1\", \"WARNING: Test for license information of bundled components is not implemented (FRSM-15-R1.1-2).\"]}, {\"id\": 16, \"metric_identifier\": \"FRSM-16-R1.1\", \"metric_name\": \"Does the software metadata record include licensing information?\", \"metric_tests\": {\"FRSM-16-R1.1-1\": {\"metric_test_name\": \"The identifier or metadata record includes licensing and copyright information.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 1, \"metric_test_status\": \"pass\"}, \"FRSM-16-R1.1-2\": {\"metric_test_name\": \"The software licensing information is in SPDX format, or other machine-readable form.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 1, \"total\": 1}, \"metric_test_maturity\": 3, \"metric_test_status\": \"pass\"}}, \"test_status\": \"pass\", \"score\": {\"earned\": 2, \"total\": 2}, \"maturity\": 3, \"output\": [{\"license\": \"MIT License\", \"osi_approved\": true, \"details_url\": \"http://spdx.org/licenses/MIT.html\"}], \"test_debug\": [\"INFO: License verification name through SPDX registry -: MIT License\", \"INFO: Found SPDX license representation -: http://spdx.org/licenses/MIT.json\", \"SUCCESS: Found SPDX license representation (spdx url, osi_approved)\", \"SUCCESS: Found licence information in metadata\", \"INFO: Will consider all SPDX licenses as community specific licenses for FRSM-16-R1.1\"]}, {\"id\": 17, \"metric_identifier\": \"FRSM-17-R1.2\", \"metric_name\": \"Does the software include provenance information that describe the development of the software?\", \"metric_tests\": {\"FRSM-17-R1.2-1\": {\"metric_test_name\": \"The software source code repository / forge includes a commit history.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-17-R1.2-2\": {\"metric_test_name\": \"The software source code repository links commits to issues / tickets.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}, \"FRSM-17-R1.2-3\": {\"metric_test_name\": \"The software project uses other tools to capture detailed machine readable provenance information.\", \"metric_test_requirements\": [], \"metric_test_score\": {\"earned\": 0, \"total\": 1}, \"metric_test_maturity\": 0, \"metric_test_status\": \"fail\"}}, \"test_status\": \"fail\", \"score\": {\"earned\": 0, \"total\": 3}, \"maturity\": 0, \"output\": {}, \"test_debug\": [\"WARNING: Test for commit history is not implemented.\", \"WARNING: Test for commit linkage to issues is not implemented.\", \"WARNING: Test for other tools capturing provenance information is not implemented.\"]}]}\n" headers: content-length: - - '24458' + - '24397' content-type: - application/json http_version: HTTP/1.1 diff --git a/tests/functional/test_evaluation.py b/tests/functional/test_evaluation.py index f4cd71dd..be0cc1f3 100644 --- a/tests/functional/test_evaluation.py +++ b/tests/functional/test_evaluation.py @@ -176,13 +176,13 @@ def test_evaluation_software(client: FlaskClient) -> None: response = client.post(valid_url, json=payload, headers=headers) assert response.status_code == HTTP_200_OK - # these are the results from 2024-05-03 + # these are the results from 2024-08-30 expected = { "score_earned": { "A": 0, "F": 0, "I": 0, - "R": 4, + "R": 5, "A1": 0, "F1": 0, "F1.1": 0, @@ -192,10 +192,10 @@ def test_evaluation_software(client: FlaskClient) -> None: "F4": 0, "I1": 0, "I2": 0, - "R1": 0, + "R1": 1, "R1.1": 4, "R1.2": 0, - "FAIR": 4.0, + "FAIR": 5, }, "score_total": { "A": 2, @@ -214,26 +214,26 @@ def test_evaluation_software(client: FlaskClient) -> None: "R1": 8, "R1.1": 5, "R1.2": 3, - "FAIR": 45.0, + "FAIR": 45, }, "score_percent": { - "A": 0.0, - "F": 0.0, - "I": 0.0, - "R": 25.0, - "A1": 0.0, - "F1": 0.0, - "F1.1": 0.0, - "F1.2": 0.0, - "F2": 0.0, - "F3": 0.0, - "F4": 0.0, - "I1": 0.0, - "I2": 0.0, - "R1": 0.0, - "R1.1": 80.0, - "R1.2": 0.0, - "FAIR": 8.89, + "A": 0, + "F": 0, + "I": 0, + "R": 31.25, + "A1": 0, + "F1": 0, + "F1.1": 0, + "F1.2": 0, + "F2": 0, + "F3": 0, + "F4": 0, + "I1": 0, + "I2": 0, + "R1": 12.5, + "R1.1": 80, + "R1.2": 0, + "FAIR": 11.11, }, "status_total": { "A1": 1, @@ -264,14 +264,14 @@ def test_evaluation_software(client: FlaskClient) -> None: "F4": 0, "I1": 0, "I2": 0, - "R1": 0, + "R1": 1, "R1.1": 2, "R1.2": 0, "A": 0, "F": 0, "I": 0, - "R": 2, - "FAIR": 2, + "R": 3, + "FAIR": 3, }, "maturity": { "A": 0, @@ -287,10 +287,10 @@ def test_evaluation_software(client: FlaskClient) -> None: "F4": 0, "I1": 0, "I2": 0, - "R1": 0, + "R1": 1, "R1.1": 3, "R1.2": 0, - "FAIR": 1.0, + "FAIR": 1, }, } response_json = response.json()