From bcbbd6f8e148f7227b191da2d6202dcb7e99aa15 Mon Sep 17 00:00:00 2001 From: Chris van Run Date: Thu, 14 Dec 2023 17:14:21 +0100 Subject: [PATCH 1/3] Add a generic load_file / write_file function --- grand_challenge_forge/generation_utils.py | 23 +++++------- .../inference.py.j2 | 36 ++++++++++++++++--- .../evaluate.py.j2 | 25 ++++++++++--- tests/utils.py | 8 ++++- 4 files changed, 67 insertions(+), 25 deletions(-) diff --git a/grand_challenge_forge/generation_utils.py b/grand_challenge_forge/generation_utils.py index 6932332..515b47d 100644 --- a/grand_challenge_forge/generation_utils.py +++ b/grand_challenge_forge/generation_utils.py @@ -19,22 +19,15 @@ def enrich_phase_context(context): "relative_path" ].endswith(".json") ci["is_image"] = ci["super_kind"] == "Image" + ci["is_file"] = ci["super_kind"] == "File" and not ci[ + "relative_path" + ].endswith(".json") - phase_context["has_input_json"] = any( - ci["is_json"] for ci in phase_context["inputs"] - ) - - phase_context["has_output_json"] = any( - ci["is_json"] for ci in phase_context["outputs"] - ) - - phase_context["has_input_image"] = any( - ci["is_image"] for ci in phase_context["inputs"] - ) - - phase_context["has_output_image"] = any( - ci["is_image"] for ci in phase_context["outputs"] - ) + for _type in ["json", "image", "file"]: + for in_out in ["input", "output"]: + phase_context[f"has_{in_out}_{_type}"] = any( + ci[f"is_{_type}"] for ci in phase_context[f"{in_out}s"] + ) def create_civ_stub_file(*, target_dir, component_interface): diff --git a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 index 78119a4..7c494dd 100644 --- a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 +++ b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 @@ -77,27 +77,33 @@ def run(): content={{ py_slug }} ) {% endif -%} + {% if ci.is_json -%} + write_file( + location=OUTPUT_PATH / "{{ ci.relative_path }}", + content={{ py_slug }} + ) + {% endif -%} {% endfor %} return 0 +{%- if cookiecutter.phase.has_input_json %} -{% if cookiecutter.phase.has_input_json -%} def load_json_file(*, location): # Reads a json file with open(location, 'r') as f: return json.loads(f.read()) {%- endif %} +{%- if cookiecutter.phase.has_output_json %} -{% if cookiecutter.phase.has_output_json -%} def write_json_file(*, location, content): # Writes a json file with open(location, 'w') as f: f.write(json.dumps(content, indent=4)) {%- endif %} +{%- if cookiecutter.phase.has_input_image %} -{% if cookiecutter.phase.has_input_image -%} def load_image_file_as_array(*, location): # Use SimpleITK to read a file input_files = glob(str(location / "*")) @@ -106,9 +112,9 @@ def load_image_file_as_array(*, location): # Convert it to a Numpy array return SimpleITK.GetArrayFromImage(result) {%- endif %} +{%- if cookiecutter.phase.has_output_image %} -{% if cookiecutter.phase.has_output_image -%} def write_array_as_image_file(*, location, array): location.mkdir(parents=True, exist_ok=True) @@ -119,6 +125,28 @@ def write_array_as_image_file(*, location, array): useCompression=True, ) {%- endif %} +{%- if cookiecutter.phase.has_input_file %} + + +# Note to the challenge hosts: +# the following function is very generic and should likely +# be adopted to something more specific for your challenge +def load_file(*, location): + # Reads the content of a file + with open(location) as f: + return f.read() +{%- endif %} +{%- if cookiecutter.phase.has_output_file %} + + +# Note to the challenge hosts: +# the following function is very generic and should likely +# be adopted to something more specific for your challenge +def write_file(*, location, content): + # Write the content to a file + with open(location, 'w') as f: + return f.write(content) +{%- endif %} def _show_torch_cuda_info(): diff --git a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 index fa5daa9..a06a68b 100644 --- a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 +++ b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 @@ -81,6 +81,9 @@ def process(job): {% if ci.is_json -%} {{ py_slug }} = load_json_file(location={{ py_slug }}_location) {% endif -%} + {% if ci.is_file -%} + {{ py_slug }} = load_file(location={{ py_slug }}_location) + {% endif -%} {% endfor -%} # Retrieve the input image name {% for ci in cookiecutter.phase.inputs -%} @@ -118,8 +121,9 @@ def read_predictions(): # The prediction file tells us the location of the users' predictions with open(INPUT_DIRECTORY / "predictions.json") as f: return json.loads(f.read()) +{%- if cookiecutter.phase.has_input_image or cookiecutter.phase.has_output_image %} + -{% if cookiecutter.phase.has_input_image or cookiecutter.phase.has_output_image%} def get_image_name(*, values, slug): # This tells us the user-provided name of the input or output image for value in values: @@ -127,7 +131,8 @@ def get_image_name(*, values, slug): return value["image"]["name"] raise RuntimeError(f"Image with interface {slug} not found!") -{% endif %} +{%- endif %} + def get_interface_relative_path(*, values, slug): # Gets the location of the interface relative to the input or output @@ -142,15 +147,17 @@ def get_file_location(*, job_pk, values, slug): # Where a job's output file will be located in the evaluation container relative_path = get_interface_relative_path(values=values, slug=slug) return INPUT_DIRECTORY / job_pk / "output" / relative_path +{%- if cookiecutter.phase.has_output_json %} + -{% if cookiecutter.phase.has_output_json %} def load_json_file(*, location): # Reads a json file with open(location) as f: return json.loads(f.read()) -{% endif %} +{%- endif %} +{%- if cookiecutter.phase.has_output_image %} + -{% if cookiecutter.phase.has_output_image -%} def load_image_file(*, location): # Use SimpleITK to read a file input_files = glob(str(location / "*")) @@ -159,6 +166,14 @@ def load_image_file(*, location): # Convert it to a Numpy array return SimpleITK.GetArrayFromImage(result) {%- endif %} +{%- if cookiecutter.phase.has_output_file %} + + +def load_file(*, location): + # Reads the content of a file + with open(location) as f: + return f.read() +{%- endif %} def write_metrics(*, metrics): diff --git a/tests/utils.py b/tests/utils.py index c70950b..c7a1418 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -29,7 +29,13 @@ "slug": "yet-another-input-ci-slug", "kind": "Anything", "super_kind": "Value", - "relative_path": "another-input-value.json" + "relative_path": "yet-another-input-value.json" + }, + { + "slug": "yet-another-non-json-input-ci-slug", + "kind": "Anything", + "super_kind": "File", + "relative_path": "yet-another-non-json-input-value" } ], "outputs": [ From b4122538dd7a656be57f9d57ebc4dd73f4dba930 Mon Sep 17 00:00:00 2001 From: Chris van Run Date: Thu, 14 Dec 2023 17:51:45 +0100 Subject: [PATCH 2/3] Add a generic load_file / write_file function --- grand_challenge_forge/generation_utils.py | 4 +- .../inference.py.j2 | 9 ++-- .../evaluate.py.j2 | 41 +++++++++++++------ tests/utils.py | 12 ++++++ 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/grand_challenge_forge/generation_utils.py b/grand_challenge_forge/generation_utils.py index 515b47d..4afe08d 100644 --- a/grand_challenge_forge/generation_utils.py +++ b/grand_challenge_forge/generation_utils.py @@ -15,9 +15,7 @@ def enrich_phase_context(context): *phase_context["inputs"], *phase_context["outputs"], ]: - ci["is_json"] = ci["kind"] == "Anything" or ci[ - "relative_path" - ].endswith(".json") + ci["is_json"] = ci["relative_path"].endswith(".json") ci["is_image"] = ci["super_kind"] == "Image" ci["is_file"] = ci["super_kind"] == "File" and not ci[ "relative_path" diff --git a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 index 7c494dd..ca44600 100644 --- a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 +++ b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 @@ -55,12 +55,13 @@ def run(): print(f.read()) # For now, let us set make bogus predictions - {% for ci in cookiecutter.phase.outputs -%} + {%- for ci in cookiecutter.phase.outputs %} {{ ci.slug | replace("-", "_")}} = {%- if ci.is_image %} numpy.eye(4, 2) - {%- else %} {"content": "should match the required format"} + {%- elif ci.is_json %} {"content": "should match the required format"} + {%- elif ci.is_file %} "content: should match the required format" {% endif %} - {% endfor -%} + {%- endfor %} # Save your output {% for ci in cookiecutter.phase.outputs -%} @@ -77,7 +78,7 @@ def run(): content={{ py_slug }} ) {% endif -%} - {% if ci.is_json -%} + {% if ci.is_file -%} write_file( location=OUTPUT_PATH / "{{ ci.relative_path }}", content={{ py_slug }} diff --git a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 index a06a68b..7933a68 100644 --- a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 +++ b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 @@ -67,42 +67,57 @@ def process(job): report += pformat(job) report += "\n" - # First read the results - {% for ci in cookiecutter.phase.outputs -%} - {% set py_slug = ci.slug | replace("-", "_") -%} + + # Firstly, find the location of the results + {% for ci in cookiecutter.phase.outputs %} + {%- set py_slug = ci.slug | replace("-", "_") -%} {{ py_slug }}_location = get_file_location( job_pk=job["pk"], values=job["outputs"], slug="{{ ci.slug }}", ) + {% endfor %} + + # Secondly, read the results + {% for ci in cookiecutter.phase.outputs -%} + {% set py_slug = ci.slug | replace("-", "_") -%} {% if ci.is_image -%} - {{ py_slug }} = load_image_file(location={{ py_slug }}_location) + {{ py_slug }} = load_image_file( + location={{ py_slug }}_location, + ) {% endif -%} {% if ci.is_json -%} - {{ py_slug }} = load_json_file(location={{ py_slug }}_location) + {{ py_slug }} = load_json_file( + location={{ py_slug }}_location, + ) {% endif -%} {% if ci.is_file -%} - {{ py_slug }} = load_file(location={{ py_slug }}_location) + {{ py_slug }} = load_file( + location={{ py_slug }}_location, + ) {% endif -%} - {% endfor -%} - # Retrieve the input image name + {%- endfor %} + + # Thirdly, retrieve the input image name {% for ci in cookiecutter.phase.inputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} {% if ci.is_image -%} {{ py_slug }}_image_name = get_image_name( values=job["inputs"], slug="{{ ci.slug }}", - ) - + ) {% endif -%} - {% endfor -%} + {%- endfor %} - # Now you would need to load your ground truth - # make sure to include it in your evaluation container + # Fourthly, your load your ground truth + # Include it in your evaluation container by placing it in ground_truth/ with open(GROUND_TRUTH_DIRECTORY / "some_resource.txt", "r") as f: report += f.read() print(report) + + + # Finally, calculate by comparing the ground truth to the actual results return { "my_metric": random.choice([1, 0]), } diff --git a/tests/utils.py b/tests/utils.py index c7a1418..040f369 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -50,6 +50,18 @@ "kind": "Anything", "super_kind": "File", "relative_path": "output-value.json" + }, + { + "slug": "yet-another-output-ci-slug", + "kind": "Anything", + "super_kind": "Value", + "relative_path": "yet-another-output-value.json" + }, + { + "slug": "yet-another-non-json-output-ci-slug", + "kind": "Anything", + "super_kind": "File", + "relative_path": "yet-another-non-json-output-value" } ] }, From 6c6afc2df01225ac2137c3779469dbad11688293 Mon Sep 17 00:00:00 2001 From: Chris van Run Date: Fri, 15 Dec 2023 08:55:29 +0100 Subject: [PATCH 3/3] Black --- .../example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 index f3b3bc4..e699798 100644 --- a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 +++ b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 @@ -97,7 +97,7 @@ def process(job): ) {% endif -%} {%- endfor %} - + # Thirdly, retrieve the input image name to match it with an image in your ground truth {% for ci in cookiecutter.phase.inputs -%}