diff --git a/README.md b/README.md index 91f2875..ee09dea 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ a directory `dist/` (default). "archive": { "url": "https://grand-challenge.org/archives/archive-slug/" }, - "inputs": [ + "algorithm_inputs": [ { "slug": "input-ci-slug", "kind": "Segmentation", @@ -62,7 +62,7 @@ a directory `dist/` (default). "relative_path": "another-input-value.json" } ], - "outputs": [ + "algorithm_outputs": [ { "slug": "output-ci-slug", "kind": "Image", @@ -82,7 +82,7 @@ a directory `dist/` (default). "archive": { "url": "https://grand-challenge.org/archives/another-archive-slug/" }, - "inputs": [ + "algorithm_inputs": [ { "slug": "input-ci-slug", "kind": "Image", @@ -90,7 +90,7 @@ a directory `dist/` (default). "relative_path": "images/input-value" } ], - "outputs": [ + "algorithm_outputs": [ { "slug": "another-output-ci-slug", "kind": "Anything", diff --git a/grand_challenge_forge/forge.py b/grand_challenge_forge/forge.py index 41e4e9b..cc3681b 100644 --- a/grand_challenge_forge/forge.py +++ b/grand_challenge_forge/forge.py @@ -111,7 +111,7 @@ def generate_upload_to_archive_script( # Map the expected case, but only create after the script expected_cases, create_files_func = _gen_expected_archive_cases( - inputs=context["phase"]["inputs"], + inputs=context["phase"]["algorithm_inputs"], output_directory=script_dir, ) context["phase"]["expected_cases"] = expected_cases @@ -174,7 +174,7 @@ def generate_example_algorithm( # Create input files input_dir = algorithm_dir / "test" / "input" - for input_ci in context["phase"]["inputs"]: + for input_ci in context["phase"]["algorithm_inputs"]: create_civ_stub_file( target_dir=input_dir / input_ci["relative_path"], component_interface=input_ci, @@ -226,9 +226,13 @@ def generate_predictions(context, evaluation_dir, n=3): predictions.append( { "pk": str(uuid.uuid4()), - "inputs": [ci_to_civ(ci) for ci in context["phase"]["inputs"]], + "inputs": [ + ci_to_civ(ci) + for ci in context["phase"]["algorithm_inputs"] + ], "outputs": [ - ci_to_civ(ci) for ci in context["phase"]["outputs"] + ci_to_civ(ci) + for ci in context["phase"]["algorithm_outputs"] ], "status": "Succeeded", "started_at": "2023-11-29T10:31:25.691799Z", diff --git a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 index 4906aa1..dcd83e8 100644 --- a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 +++ b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 @@ -18,10 +18,10 @@ Any container that shows the same behavior will do, this is purely an example of Happy programming! """ from pathlib import Path -{% if cookiecutter.phase.inputs | has_json or cookiecutter.phase.outputs | has_json -%} +{% if cookiecutter.phase.algorithm_inputs | has_json or cookiecutter.phase.algorithm_outputs | has_json -%} import json {%- endif %} -{% if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image -%} +{% if cookiecutter.phase.algorithm_inputs | has_image or cookiecutter.phase.algorithm_outputs | has_image -%} from glob import glob import SimpleITK import numpy @@ -35,7 +35,7 @@ RESOURCE_PATH = Path("resources") def run(): # Read the input - {% for ci in cookiecutter.phase.inputs -%} + {% for ci in cookiecutter.phase.algorithm_inputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} {% if ci | is_image -%} {{ py_slug }} = load_image_file_as_array( @@ -55,7 +55,7 @@ def run(): print(f.read()) # For now, let us set make bogus predictions - {%- for ci in cookiecutter.phase.outputs %} + {%- for ci in cookiecutter.phase.algorithm_outputs %} {{ ci.slug | replace("-", "_")}} = {%- if ci | is_image %} numpy.eye(4, 2) {%- elif ci | is_json %} {"content": "should match the required format"} @@ -64,7 +64,7 @@ def run(): {%- endfor %} # Save your output - {% for ci in cookiecutter.phase.outputs -%} + {% for ci in cookiecutter.phase.algorithm_outputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} {% if ci | is_image -%} write_array_as_image_file( @@ -86,7 +86,7 @@ def run(): {% endif -%} {% endfor %} return 0 -{%- if cookiecutter.phase.inputs | has_json %} +{%- if cookiecutter.phase.algorithm_inputs | has_json %} def load_json_file(*, location): @@ -94,7 +94,7 @@ def load_json_file(*, location): with open(location, 'r') as f: return json.loads(f.read()) {%- endif %} -{%- if cookiecutter.phase.outputs | has_json %} +{%- if cookiecutter.phase.algorithm_outputs | has_json %} def write_json_file(*, location, content): @@ -102,7 +102,7 @@ def write_json_file(*, location, content): with open(location, 'w') as f: f.write(json.dumps(content, indent=4)) {%- endif %} -{%- if cookiecutter.phase.inputs | has_image %} +{%- if cookiecutter.phase.algorithm_inputs | has_image %} def load_image_file_as_array(*, location): @@ -113,7 +113,7 @@ def load_image_file_as_array(*, location): # Convert it to a Numpy array return SimpleITK.GetArrayFromImage(result) {%- endif %} -{%- if cookiecutter.phase.outputs | has_image %} +{%- if cookiecutter.phase.algorithm_outputs | has_image %} def write_array_as_image_file(*, location, array): @@ -129,7 +129,7 @@ def write_array_as_image_file(*, location, array): useCompression=True, ) {%- endif %} -{%- if cookiecutter.phase.inputs | has_file %} +{%- if cookiecutter.phase.algorithm_inputs | has_file %} # Note to the challenge hosts: @@ -140,7 +140,7 @@ def load_file(*, location): with open(location) as f: return f.read() {%- endif %} -{%- if cookiecutter.phase.outputs | has_file %} +{%- if cookiecutter.phase.algorithm_outputs | has_file %} # Note to the challenge hosts: diff --git a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 index 97a0c26..e1bc465 100644 --- a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 +++ b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 @@ -1,4 +1,4 @@ -{% if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image -%} +{% if cookiecutter.phase.algorithm_inputs | has_image or cookiecutter.phase.algorithm_outputs | has_image -%} SimpleITK numpy {%- endif %} diff --git a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 index 96ecb7a..8c59842 100644 --- a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 +++ b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 @@ -18,7 +18,7 @@ Any container that shows the same behavior will do, this is purely an example of Happy programming! """ import json -{% if cookiecutter.phase.inputs | has_image -%} +{% if cookiecutter.phase.algorithm_inputs | has_image -%} from glob import glob import SimpleITK {%- endif %} @@ -69,7 +69,7 @@ def process(job): # Firstly, find the location of the results - {% for ci in cookiecutter.phase.outputs %} + {% for ci in cookiecutter.phase.algorithm_outputs %} {%- set py_slug = ci.slug | replace("-", "_") -%} {{ py_slug }}_location = get_file_location( job_pk=job["pk"], @@ -79,7 +79,7 @@ def process(job): {% endfor %} # Secondly, read the results - {% for ci in cookiecutter.phase.outputs -%} + {% for ci in cookiecutter.phase.algorithm_outputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} {% if ci | is_image -%} {{ py_slug }} = load_image_file( @@ -100,7 +100,7 @@ def process(job): # Thirdly, retrieve the input image name to match it with an image in your ground truth - {% for ci in cookiecutter.phase.inputs -%} + {% for ci in cookiecutter.phase.algorithm_inputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} {% if ci | is_image -%} {{ py_slug }}_image_name = get_image_name( @@ -138,7 +138,7 @@ def read_predictions(): # The prediction file tells us the location of the users' predictions with open(INPUT_DIRECTORY / "predictions.json") as f: return json.loads(f.read()) -{%- if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image %} +{%- if cookiecutter.phase.algorithm_inputs | has_image or cookiecutter.phase.algorithm_outputs | has_image %} def get_image_name(*, values, slug): @@ -164,7 +164,7 @@ def get_file_location(*, job_pk, values, slug): # Where a job's output file will be located in the evaluation container relative_path = get_interface_relative_path(values=values, slug=slug) return INPUT_DIRECTORY / job_pk / "output" / relative_path -{%- if cookiecutter.phase.outputs | has_json %} +{%- if cookiecutter.phase.algorithm_outputs | has_json %} def load_json_file(*, location): @@ -172,7 +172,7 @@ def load_json_file(*, location): with open(location) as f: return json.loads(f.read()) {%- endif %} -{%- if cookiecutter.phase.outputs | has_image %} +{%- if cookiecutter.phase.algorithm_outputs | has_image %} def load_image_file(*, location): @@ -183,7 +183,7 @@ def load_image_file(*, location): # Convert it to a Numpy array return SimpleITK.GetArrayFromImage(result) {%- endif %} -{%- if cookiecutter.phase.outputs | has_file %} +{%- if cookiecutter.phase.algorithm_outputs | has_file %} def load_file(*, location): diff --git a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 index 97a0c26..e1bc465 100644 --- a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 +++ b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 @@ -1,4 +1,4 @@ -{% if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image -%} +{% if cookiecutter.phase.algorithm_inputs | has_image or cookiecutter.phase.algorithm_outputs | has_image -%} SimpleITK numpy {%- endif %} diff --git a/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 b/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 index 70d6316..1d0365b 100644 --- a/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 +++ b/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 @@ -33,7 +33,7 @@ Happy uploading! from pathlib import Path import gcapi -{% if cookiecutter.phase.inputs | has_json -%} +{% if cookiecutter.phase.algorithm_inputs | has_json -%} import json {%- endif %} @@ -42,7 +42,7 @@ API_TOKEN = "REPLACE-ME-WITH-YOUR-TOKEN" ARCHIVE_SLUG = "{{ cookiecutter.phase.archive.slug }}" EXPECTED_CASES = [ - # for: {% for ci in cookiecutter.phase.inputs -%}{{ ci.slug }}{%- if not loop.last -%}, {% endif -%}{% endfor %} + # for: {% for ci in cookiecutter.phase.algorithm_inputs -%}{{ ci.slug }}{%- if not loop.last -%}, {% endif -%}{% endfor %} {%- for expected_cases in cookiecutter.phase.expected_cases %} {{ expected_cases }}, {%- endfor %} @@ -86,7 +86,7 @@ def upload_files(): def map_case_content_to_interfaces(case): return { - {%- for ci in cookiecutter.phase.inputs %} + {%- for ci in cookiecutter.phase.algorithm_inputs %} {%- if ci | is_json %} "{{ ci.slug }}": json.loads(Path(case[{{ loop.index - 1}}]).read_text()), {%- else %} diff --git a/grand_challenge_forge/quality_control.py b/grand_challenge_forge/quality_control.py index 482e030..111e2fd 100644 --- a/grand_challenge_forge/quality_control.py +++ b/grand_challenge_forge/quality_control.py @@ -75,7 +75,7 @@ def _test_example_algorithm(phase_context, algorithm_dir, number_run): _test_subprocess(script_dir=algorithm_dir, number_run=number_run) # Check if output is generated (ignore content) - for output in phase_context["phase"]["outputs"]: + for output in phase_context["phase"]["algorithm_outputs"]: expected_file = output_dir / output["relative_path"] if not expected_file.exists(): raise QualityFailureError( diff --git a/grand_challenge_forge/schemas.py b/grand_challenge_forge/schemas.py index f20c566..5f7e963 100644 --- a/grand_challenge_forge/schemas.py +++ b/grand_challenge_forge/schemas.py @@ -48,16 +48,21 @@ "properties": { "slug": {"type": "string"}, "archive": ARCHIVE_SCHEMA, - "inputs": { + "algorithm_inputs": { "type": "array", "items": COMPONENT_INTERFACE_SCHEMA, }, - "outputs": { + "algorithm_outputs": { "type": "array", "items": COMPONENT_INTERFACE_SCHEMA, }, }, - "required": ["slug", "archive", "inputs", "outputs"], + "required": [ + "slug", + "archive", + "algorithm_inputs", + "algorithm_outputs", + ], "additionalProperties": True, # Allow additional properties }, }, diff --git a/tests/utils.py b/tests/utils.py index 6c1e2f5..f675409 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -27,7 +27,7 @@ "slug": "archive-slug", "url": "https://grand-challenge.org/archives/archive-slug/" }, - "inputs": [ + "algorithm_inputs": [ { "slug": "input-ci-slug", "kind": "Segmentation", @@ -53,7 +53,7 @@ "relative_path": "yet-another-non-json-input-value" } ], - "outputs": [ + "algorithm_outputs": [ { "slug": "output-ci-slug", "kind": "Image", @@ -86,7 +86,7 @@ "slug": "another-archive-slug", "url": "https://grand-challenge.org/archives/another-archive-slug/" }, - "inputs": [ + "algorithm_inputs": [ { "slug": "input-ci-slug", "kind": "Image", @@ -94,7 +94,7 @@ "relative_path": "images/input-value" } ], - "outputs": [ + "algorithm_outputs": [ { "slug": "another-output-ci-slug", "kind": "Anything",