From f330c702291feb68ba1beac36dcd6da4af042f43 Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Fri, 7 Jul 2023 12:02:32 +0300 Subject: [PATCH 01/15] create v1.0.1 --- v1.0.1/admin-schema.json | 653 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 653 insertions(+) create mode 100644 v1.0.1/admin-schema.json diff --git a/v1.0.1/admin-schema.json b/v1.0.1/admin-schema.json new file mode 100644 index 0000000..20edf5c --- /dev/null +++ b/v1.0.1/admin-schema.json @@ -0,0 +1,653 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v1.0.1/admin-schema.json", + "title": "Schema for Modeling Hub administrative settings", + "description": "This JSON file provides a schema for modeling hub administrative information.", + "type": "object", + "properties": { + "schema_version": { + "description": "URL to a version of the Modeling Hub schema admin-schema.json file (see https://github.com/Infectious-Disease-Modeling-Hubs/schemas). Used to declare the schema version an 'admin.json' file is written for and for config file validation. The URL provided should be the URL to the raw content of the schema file on GitHub.", + "examples": ["https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v0.0.1/admin-schema.json"], + "type": "string", + "format": "uri" + }, + "name": { + "description": "The name of the hub.", + "type": "string", + "examples": ["US COVID-19 Forecast Hub​"] + }, + "maintainer": { + "description": "The entity that maintains and runs the hub.", + "type": "string", + "examples": ["The Consortium of Infectious Disease Modeling Hubs"] + }, + "contact": { + "description": "The name and email of a human being who serves as a point of contact for the hub.", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + } + } + }, + "repository_host": { + "description": "The name of the web host for the hub repository", + "type": "string", + "enum": [ + "GitHub" + ] + }, + "repository_url": { + "description": "The url for the hub repository.", + "type": "string", + "examples": ["https://github.com/reichlab/covid19-forecast-hub"] + }, + "zoltar_project_id": { + "description": "The project id of the Hub in Zoltar", + "type": "integer", + "examples": [44] + }, + "hub_models": { + "description": "Array of ensemble and baseline models produced by the hub", + "type": "array", + "items": { + "type": "object", + "properties": { + "team_abbr": { + "description": "Abbreviated team name of the hub", + "type": "string", + "pattern": "^[a-zA-Z0-9_+]+$", + "maxLength": 16 + }, + "model_abbr": { + "description": "Abbreviated name of the model", + "type": "string", + "pattern": "^[a-zA-Z0-9_+]+$", + "maxLength": 16 + }, + "model_type": { + "description": "The type of model: baseline or ensemble", + "type": "string", + "enum": [ + "baseline", + "ensemble" + ] + } + } + } + }, + "citation": { + "description": "One or more citations for the hub.", + "type": "string", + "examples": ["Cramer, E.Y., Huang, Y., Wang, Y. et al. The United States COVID-19 Forecast Hub dataset. Sci Data 9, 462 (2022). https://doi.org/10.1038/s41597-022-01517-w"] + }, + "file_format": { + "description": "Accepted file formats of model output files.", + "examples": [ + ["arrow", "parquet"], + ["csv"] + ], + "type": "array", + "items": { + "type": "string", + "enum": [ + "csv", + "parquet", + "arrow" + ] + } + }, + "timezone": { + "description": "Hub timezone in TZ database name format. Used primarily for determining exact time of submission deadlines.", + "examples": [ + "Africa/Cairo", + "America/Sao_Paulo", + "Asia/Bangkok", + "US/Eastern", + "Europe/Berlin" + ], + "type": "string", + "enum": [ + "Africa/Abidjan", + "Africa/Accra", + "Africa/Addis_Ababa", + "Africa/Algiers", + "Africa/Asmara", + "Africa/Asmera", + "Africa/Bamako", + "Africa/Bangui", + "Africa/Banjul", + "Africa/Bissau", + "Africa/Blantyre", + "Africa/Brazzaville", + "Africa/Bujumbura", + "Africa/Cairo", + "Africa/Casablanca", + "Africa/Ceuta", + "Africa/Conakry", + "Africa/Dakar", + "Africa/Dar_es_Salaam", + "Africa/Djibouti", + "Africa/Douala", + "Africa/El_Aaiun", + "Africa/Freetown", + "Africa/Gaborone", + "Africa/Harare", + "Africa/Johannesburg", + "Africa/Juba", + "Africa/Kampala", + "Africa/Khartoum", + "Africa/Kigali", + "Africa/Kinshasa", + "Africa/Lagos", + "Africa/Libreville", + "Africa/Lome", + "Africa/Luanda", + "Africa/Lubumbashi", + "Africa/Lusaka", + "Africa/Malabo", + "Africa/Maputo", + "Africa/Maseru", + "Africa/Mbabane", + "Africa/Mogadishu", + "Africa/Monrovia", + "Africa/Nairobi", + "Africa/Ndjamena", + "Africa/Niamey", + "Africa/Nouakchott", + "Africa/Ouagadougou", + "Africa/Porto-Novo", + "Africa/Sao_Tome", + "Africa/Timbuktu", + "Africa/Tripoli", + "Africa/Tunis", + "Africa/Windhoek", + "America/Adak", + "America/Anchorage", + "America/Anguilla", + "America/Antigua", + "America/Araguaina", + "America/Argentina/Buenos_Aires", + "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", + "America/Argentina/Cordoba", + "America/Argentina/Jujuy", + "America/Argentina/La_Rioja", + "America/Argentina/Mendoza", + "America/Argentina/Rio_Gallegos", + "America/Argentina/Salta", + "America/Argentina/San_Juan", + "America/Argentina/San_Luis", + "America/Argentina/Tucuman", + "America/Argentina/Ushuaia", + "America/Aruba", + "America/Asuncion", + "America/Atikokan", + "America/Atka", + "America/Bahia", + "America/Bahia_Banderas", + "America/Barbados", + "America/Belem", + "America/Belize", + "America/Blanc-Sablon", + "America/Boa_Vista", + "America/Bogota", + "America/Boise", + "America/Buenos_Aires", + "America/Cambridge_Bay", + "America/Campo_Grande", + "America/Cancun", + "America/Caracas", + "America/Catamarca", + "America/Cayenne", + "America/Cayman", + "America/Chicago", + "America/Chihuahua", + "America/Coral_Harbour", + "America/Cordoba", + "America/Costa_Rica", + "America/Creston", + "America/Cuiaba", + "America/Curacao", + "America/Danmarkshavn", + "America/Dawson", + "America/Dawson_Creek", + "America/Denver", + "America/Detroit", + "America/Dominica", + "America/Edmonton", + "America/Eirunepe", + "America/El_Salvador", + "America/Ensenada", + "America/Fort_Nelson", + "America/Fort_Wayne", + "America/Fortaleza", + "America/Glace_Bay", + "America/Godthab", + "America/Goose_Bay", + "America/Grand_Turk", + "America/Grenada", + "America/Guadeloupe", + "America/Guatemala", + "America/Guayaquil", + "America/Guyana", + "America/Halifax", + "America/Havana", + "America/Hermosillo", + "America/Indiana/Indianapolis", + "America/Indiana/Knox", + "America/Indiana/Marengo", + "America/Indiana/Petersburg", + "America/Indiana/Tell_City", + "America/Indiana/Vevay", + "America/Indiana/Vincennes", + "America/Indiana/Winamac", + "America/Indianapolis", + "America/Inuvik", + "America/Iqaluit", + "America/Jamaica", + "America/Jujuy", + "America/Juneau", + "America/Kentucky/Louisville", + "America/Kentucky/Monticello", + "America/Knox_IN", + "America/Kralendijk", + "America/La_Paz", + "America/Lima", + "America/Los_Angeles", + "America/Louisville", + "America/Lower_Princes", + "America/Maceio", + "America/Managua", + "America/Manaus", + "America/Marigot", + "America/Martinique", + "America/Matamoros", + "America/Mazatlan", + "America/Mendoza", + "America/Menominee", + "America/Merida", + "America/Metlakatla", + "America/Mexico_City", + "America/Miquelon", + "America/Moncton", + "America/Monterrey", + "America/Montevideo", + "America/Montreal", + "America/Montserrat", + "America/Nassau", + "America/New_York", + "America/Nipigon", + "America/Nome", + "America/Noronha", + "America/North_Dakota/Beulah", + "America/North_Dakota/Center", + "America/North_Dakota/New_Salem", + "America/Nuuk", + "America/Ojinaga", + "America/Panama", + "America/Pangnirtung", + "America/Paramaribo", + "America/Phoenix", + "America/Port_of_Spain", + "America/Port-au-Prince", + "America/Porto_Acre", + "America/Porto_Velho", + "America/Puerto_Rico", + "America/Punta_Arenas", + "America/Rainy_River", + "America/Rankin_Inlet", + "America/Recife", + "America/Regina", + "America/Resolute", + "America/Rio_Branco", + "America/Rosario", + "America/Santa_Isabel", + "America/Santarem", + "America/Santiago", + "America/Santo_Domingo", + "America/Sao_Paulo", + "America/Scoresbysund", + "America/Shiprock", + "America/Sitka", + "America/St_Barthelemy", + "America/St_Johns", + "America/St_Kitts", + "America/St_Lucia", + "America/St_Thomas", + "America/St_Vincent", + "America/Swift_Current", + "America/Tegucigalpa", + "America/Thule", + "America/Thunder_Bay", + "America/Tijuana", + "America/Toronto", + "America/Tortola", + "America/Vancouver", + "America/Virgin", + "America/Whitehorse", + "America/Winnipeg", + "America/Yakutat", + "America/Yellowknife", + "Antarctica/Casey", + "Antarctica/Davis", + "Antarctica/DumontDUrville", + "Antarctica/Macquarie", + "Antarctica/Mawson", + "Antarctica/McMurdo", + "Antarctica/Palmer", + "Antarctica/Rothera", + "Antarctica/South_Pole", + "Antarctica/Syowa", + "Antarctica/Troll", + "Antarctica/Vostok", + "Arctic/Longyearbyen", + "Asia/Aden", + "Asia/Almaty", + "Asia/Amman", + "Asia/Anadyr", + "Asia/Aqtau", + "Asia/Aqtobe", + "Asia/Ashgabat", + "Asia/Ashkhabad", + "Asia/Atyrau", + "Asia/Baghdad", + "Asia/Bahrain", + "Asia/Baku", + "Asia/Bangkok", + "Asia/Barnaul", + "Asia/Beirut", + "Asia/Bishkek", + "Asia/Brunei", + "Asia/Calcutta", + "Asia/Chita", + "Asia/Choibalsan", + "Asia/Chongqing", + "Asia/Chungking", + "Asia/Colombo", + "Asia/Dacca", + "Asia/Damascus", + "Asia/Dhaka", + "Asia/Dili", + "Asia/Dubai", + "Asia/Dushanbe", + "Asia/Famagusta", + "Asia/Gaza", + "Asia/Harbin", + "Asia/Hebron", + "Asia/Ho_Chi_Minh", + "Asia/Hong_Kong", + "Asia/Hovd", + "Asia/Irkutsk", + "Asia/Istanbul", + "Asia/Jakarta", + "Asia/Jayapura", + "Asia/Jerusalem", + "Asia/Kabul", + "Asia/Kamchatka", + "Asia/Karachi", + "Asia/Kashgar", + "Asia/Kathmandu", + "Asia/Katmandu", + "Asia/Khandyga", + "Asia/Kolkata", + "Asia/Krasnoyarsk", + "Asia/Kuala_Lumpur", + "Asia/Kuching", + "Asia/Kuwait", + "Asia/Macao", + "Asia/Macau", + "Asia/Magadan", + "Asia/Makassar", + "Asia/Manila", + "Asia/Muscat", + "Asia/Nicosia", + "Asia/Novokuznetsk", + "Asia/Novosibirsk", + "Asia/Omsk", + "Asia/Oral", + "Asia/Phnom_Penh", + "Asia/Pontianak", + "Asia/Pyongyang", + "Asia/Qatar", + "Asia/Qostanay", + "Asia/Qyzylorda", + "Asia/Rangoon", + "Asia/Riyadh", + "Asia/Saigon", + "Asia/Sakhalin", + "Asia/Samarkand", + "Asia/Seoul", + "Asia/Shanghai", + "Asia/Singapore", + "Asia/Srednekolymsk", + "Asia/Taipei", + "Asia/Tashkent", + "Asia/Tbilisi", + "Asia/Tehran", + "Asia/Tel_Aviv", + "Asia/Thimbu", + "Asia/Thimphu", + "Asia/Tokyo", + "Asia/Tomsk", + "Asia/Ujung_Pandang", + "Asia/Ulaanbaatar", + "Asia/Ulan_Bator", + "Asia/Urumqi", + "Asia/Ust-Nera", + "Asia/Vientiane", + "Asia/Vladivostok", + "Asia/Yakutsk", + "Asia/Yangon", + "Asia/Yekaterinburg", + "Asia/Yerevan", + "Atlantic/Azores", + "Atlantic/Bermuda", + "Atlantic/Canary", + "Atlantic/Cape_Verde", + "Atlantic/Faeroe", + "Atlantic/Faroe", + "Atlantic/Jan_Mayen", + "Atlantic/Madeira", + "Atlantic/Reykjavik", + "Atlantic/South_Georgia", + "Atlantic/St_Helena", + "Atlantic/Stanley", + "Australia/ACT", + "Australia/Adelaide", + "Australia/Brisbane", + "Australia/Broken_Hill", + "Australia/Canberra", + "Australia/Currie", + "Australia/Darwin", + "Australia/Eucla", + "Australia/Hobart", + "Australia/LHI", + "Australia/Lindeman", + "Australia/Lord_Howe", + "Australia/Melbourne", + "Australia/North", + "Australia/NSW", + "Australia/Perth", + "Australia/Queensland", + "Australia/South", + "Australia/Sydney", + "Australia/Tasmania", + "Australia/Victoria", + "Australia/West", + "Australia/Yancowinna", + "Brazil/Acre", + "Brazil/DeNoronha", + "Brazil/East", + "Brazil/West", + "Canada/Atlantic", + "Canada/Central", + "Canada/Eastern", + "Canada/Mountain", + "Canada/Newfoundland", + "Canada/Pacific", + "Canada/Saskatchewan", + "Canada/Yukon", + "Chile/Continental", + "Chile/EasterIsland", + "Europe/Amsterdam", + "Europe/Andorra", + "Europe/Astrakhan", + "Europe/Athens", + "Europe/Belfast", + "Europe/Belgrade", + "Europe/Berlin", + "Europe/Bratislava", + "Europe/Brussels", + "Europe/Bucharest", + "Europe/Budapest", + "Europe/Busingen", + "Europe/Chisinau", + "Europe/Copenhagen", + "Europe/Dublin", + "Europe/Gibraltar", + "Europe/Guernsey", + "Europe/Helsinki", + "Europe/Isle_of_Man", + "Europe/Istanbul", + "Europe/Jersey", + "Europe/Kaliningrad", + "Europe/Kiev", + "Europe/Kirov", + "Europe/Kyiv", + "Europe/Lisbon", + "Europe/Ljubljana", + "Europe/London", + "Europe/Luxembourg", + "Europe/Madrid", + "Europe/Malta", + "Europe/Mariehamn", + "Europe/Minsk", + "Europe/Monaco", + "Europe/Moscow", + "Europe/Nicosia", + "Europe/Oslo", + "Europe/Paris", + "Europe/Podgorica", + "Europe/Prague", + "Europe/Riga", + "Europe/Rome", + "Europe/Samara", + "Europe/San_Marino", + "Europe/Sarajevo", + "Europe/Saratov", + "Europe/Simferopol", + "Europe/Skopje", + "Europe/Sofia", + "Europe/Stockholm", + "Europe/Tallinn", + "Europe/Tirane", + "Europe/Tiraspol", + "Europe/Ulyanovsk", + "Europe/Uzhgorod", + "Europe/Vaduz", + "Europe/Vatican", + "Europe/Vienna", + "Europe/Vilnius", + "Europe/Volgograd", + "Europe/Warsaw", + "Europe/Zagreb", + "Europe/Zaporozhye", + "Europe/Zurich", + "Indian/Antananarivo", + "Indian/Chagos", + "Indian/Christmas", + "Indian/Cocos", + "Indian/Comoro", + "Indian/Kerguelen", + "Indian/Mahe", + "Indian/Maldives", + "Indian/Mauritius", + "Indian/Mayotte", + "Indian/Reunion", + "Mexico/BajaNorte", + "Mexico/BajaSur", + "Mexico/General", + "Pacific/Apia", + "Pacific/Auckland", + "Pacific/Bougainville", + "Pacific/Chatham", + "Pacific/Chuuk", + "Pacific/Easter", + "Pacific/Efate", + "Pacific/Enderbury", + "Pacific/Fakaofo", + "Pacific/Fiji", + "Pacific/Funafuti", + "Pacific/Galapagos", + "Pacific/Gambier", + "Pacific/Guadalcanal", + "Pacific/Guam", + "Pacific/Honolulu", + "Pacific/Johnston", + "Pacific/Kanton", + "Pacific/Kiritimati", + "Pacific/Kosrae", + "Pacific/Kwajalein", + "Pacific/Majuro", + "Pacific/Marquesas", + "Pacific/Midway", + "Pacific/Nauru", + "Pacific/Niue", + "Pacific/Norfolk", + "Pacific/Noumea", + "Pacific/Pago_Pago", + "Pacific/Palau", + "Pacific/Pitcairn", + "Pacific/Pohnpei", + "Pacific/Ponape", + "Pacific/Port_Moresby", + "Pacific/Rarotonga", + "Pacific/Saipan", + "Pacific/Samoa", + "Pacific/Tahiti", + "Pacific/Tarawa", + "Pacific/Tongatapu", + "Pacific/Truk", + "Pacific/Wake", + "Pacific/Wallis", + "Pacific/Yap", + "US/Alaska", + "US/Aleutian", + "US/Arizona", + "US/Central", + "US/East-Indiana", + "US/Eastern", + "US/Hawaii", + "US/Indiana-Starke", + "US/Michigan", + "US/Mountain", + "US/Pacific", + "US/Samoa" + ] + }, + "model_output_dir": { + "description": "Relative path, with respect to the root of hub, to directory containing model output data. If property not supplied, the default path `model_output/` assumed", + "default": "model_output", + "examples": [ + "forecasts", + "data/model_output" + ], + "type": "string" + } + }, + "required": [ + "schema_version", + "name", + "maintainer", + "contact", + "repository_url", + "file_format", + "timezone" + ], + "additionalProperties": false +} From bb06d677c867953242ea41c705f8e3bc6e6648f9 Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Fri, 7 Jul 2023 12:03:32 +0300 Subject: [PATCH 02/15] add uniqueItems check. Resolves #43 --- v1.0.1/tasks-schema.json | 1228 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1228 insertions(+) create mode 100644 v1.0.1/tasks-schema.json diff --git a/v1.0.1/tasks-schema.json b/v1.0.1/tasks-schema.json new file mode 100644 index 0000000..920805b --- /dev/null +++ b/v1.0.1/tasks-schema.json @@ -0,0 +1,1228 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v1.0.1/tasks-schema.json", + "title": "Schema for Modeling Hub model task definitions", + "description": "This is the schema of the tasks.json configuration file that defines the tasks within a modeling hub.", + "type": "object", + "properties": { + "schema_version": { + "description": "URL to a version of the Modeling Hub schema tasks-schema.json file (see https://github.com/Infectious-Disease-Modeling-Hubs/schemas). Used to declare the schema version a 'tasks.json' file is written for and for config file validation. The URL provided should be the URL to the raw content of the schema file on GitHub.", + "examples": ["https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v0.0.1/tasks-schema.json"], + "type": "string", + "format": "uri" + }, + "rounds": { + "description": "Array of modeling round properties", + "type": "array", + "items": { + "type": "object", + "description": "Individual modeling round properties", + "properties": { + "round_id_from_variable": { + "description": "Whether the round identifier is encoded by a task id variable in the data.", + "type": "boolean" + }, + "round_id": { + "description": "Round identifier. If round_id_from_variable = true, round_id should be the name of a task id variable present in all sets of modeling task specifications", + "examples": [ + "round-1", + "2022-11-05", + "origin_date" + ], + "type": "string" + }, + "round_name": { + "description": "An optional round name. This can be useful for internal referencing of rounds, for examples, when a date is used as round_id but hub maintainers and teams also refer to rounds as round-1, round-2 etc.", + "examples": [ + "round-1" + ], + "type": "string" + }, + "model_tasks": { + "type": "array", + "description": "Array defining round-specific modeling tasks. Can contain one or more groups of modeling tasks per round where each group is defined by a distinct combination of values of task id variables.", + "items": { + "type": "object", + "properties": { + "task_ids": { + "description": "Group of valid values of task id variables. A set of valid tasks corresponding to this group is formed by taking all combinations of these values.", + "type": "object", + "properties": { + "origin_date": { + "description": "An object containing arrays of required and optional unique origin dates. Origin date defines the starting point that can be used for calculating a target_date via the formula target_date = origin_date + horizon x time_units_per_horizon (e.g., with weekly data, target_date is calculated as origin_date + horizon x 7 days)", + "examples": [{ + "required": null, + "optional": [ + "2022-11-05", + "2022-11-12", + "2022-11-19" + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of origin date unique identifiers that must be present for submission to be valid. Can be null if no origin dates are required and all valid origin dates are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + }, + "optional": { + "description": "Array of valid but not required unique origin date identifiers. Can be null if all origin dates are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "scenario_id": { + "description": "An object containing arrays of required and optional unique identifiers of each valid scenario.", + "examples": [{ + "required": null, + "optional": [ + 1, + 2, + 3, + 4 + ] + }, + { + "required": null, + "optional": [ + "A-2021-03-28", + "B-2021-03-28", + "A-2021-04-05", + "B-2021-04-05" + ] + } + ], + "type": "object", + "properties": { + "required": { + "description": "Array of identifiers of scenarios that must be present in a valid submission. Can be null if no scenario ids are required and all valid ids are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": [ + "integer", + "string" + ] + } + }, + "optional": { + "description": "Array of identifiers of valid but not required scenarios. Can be null if all scenarios are required and are specified in the required property.", + "type": [ + "null", + "array" + ], + "uniqueItems": true, + "items": { + "type": [ + "integer", + "string" + ] + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "location": { + "description": "An object containing arrays of required and optional unique identifiers for each valid location, e.g. country codes, FIPS state or county level code etc.", + "examples": [{ + "required": "US", + "optional": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of location unique identifiers that must be present for submission to be valid. Can be null if no locations are required and all valid locations are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "optional": { + "description": "Array of valid but not required unique location identifiers. Can be null if all locations are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "target": { + "description": "An object containing arrays of required and optional unique identifiers for each valid target.", + "type": "object", + "examples": [{ + "required": null, + "optional": [ + "weekly rate" + ] + }], + "properties": { + "required": { + "description": "Array of target unique identifiers that must be present for submission to be valid. Can be null if no targets are required and all valid targets are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "optional": { + "description": "Array of valid but not required unique target identifiers. Can be null if all targets are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "target_date": { + "description": "An object containing arrays of required and optional unique target dates. For short-term forecasts, the target_date specifies the date of occurrence of the outcome of interest. For instance, if models are requested to forecast the number of hospitalizations that will occur on 2022-07-15, the target_date is 2022-07-15", + "examples": [{ + "required": null, + "optional": [ + "2022-11-12", + "2022-11-19", + "2022-11-26" + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of target date unique identifiers that must be present for submission to be valid. Can be null if no target dates are required and all valid target dates are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + }, + "optional": { + "description": "Array of valid but not required unique target date identifiers. Can be null if all target dates are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "horizon": { + "description": "An object containing arrays of required and optional unique horizons. Horizons define the difference between the target_date and the origin_date in time units specified by the hub (e.g., may be days, weeks, or months)", + "examples": [{ + "required": null, + "optional": [ + 1, + 2, + 3, + 4 + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of horizon unique identifiers that must be present for submission to be valid. Can be null if no horizons are required and all valid horizons are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": [ + "integer", + "string" + ] + } + }, + "optional": { + "description": "Array of valid but not required unique horizon identifiers. Can be null if all horizons are required and are specified in the required property.", + "type": [ + "null", + "array" + ], + "uniqueItems": true, + "items": { + "type": [ + "integer", + "string" + ] + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "age_group": { + "type": "object", + "description": "An object containing arrays of required and optional unique identifiers for age groups", + "examples": [{ + "required": [ + "0-5", + "6-18", + "19-24", + "25-64", + "65+" + ], + "optional": null + }], + "properties": { + "required": { + "description": "Array of age group unique identifiers that must be present for submission to be valid. Can be null if no age groups are required and all valid age groups are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "optional": { + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + } + }, + "required": [ + "required", + "optional" + ] + } + }, + "additionalProperties": true + }, + "output_type": { + "type": "object", + "description": "Object defining valid model output types for a given modeling task. The name of each property corresponds to valid values in column 'type' while the 'type_id' property of each output type defines the valid values of the 'type_id' column and the 'value' property defines the valid values of the 'value' column for a given output type.", + "properties": { + "mean": { + "type": "object", + "description": "Object defining the mean of the predictive distribution output type.", + "properties": { + "type_id": { + "description": "type_id is not meaningful for a mean output_type. The property is primarily used to determine whether mean is a required or optional output type through properties required and optional. If mean is a required output type, the required property must be an array containing the single string element 'NA' and the optional property must be set to null. If mean is an optional output type, the optional property must be an array containing the single string element 'NA' and the required property must be set to null", + "examples": [{ + "required": [ + "NA" + ], + "optional": null + }, + { + "required": null, + "optional": [ + "NA" + ] + } + ], + "type": "object", + "oneOf": [{ + "properties": { + "required": { + "description": "When mean is required, property set to single element 'NA' array", + "type": "array", + "items": { + "const": "NA", + "maxItems": 1 + } + }, + "optional": { + "description": "When mean is required, property set to null", + "type": "null" + } + } + }, + { + "properties": { + "required": { + "description": "When mean is optional, property set to null", + "type": "null" + }, + "optional": { + "description": "When mean is optional, property set to single element 'NA' array", + "type": "array", + "items": { + "const": "NA", + "maxItems": 1 + } + } + } + } + ], + "required": [ + "required", + "optional" + ] + }, + "value": { + "type": "object", + "description": "Object defining the characteristics of valid mean values.", + "examples": [{ + "type": "double", + "minimum": 0 + }], + "properties": { + "type": { + "description": "Data type of mean values.", + "type": "string", + "enum": [ + "double", + "integer" + ] + }, + "minimum": { + "description": "The minimum inclusive valid mean value", + "type": [ + "number", + "integer" + ] + }, + "maximum": { + "description": "the maximum inclusive valid mean value", + "type": [ + "number", + "integer" + ] + } + }, + "required": [ + "type" + ] + } + }, + "required": [ + "type_id", + "value" + ] + }, + "median": { + "type": "object", + "description": "Object defining the median of the predictive distribution output type", + "properties": { + "type_id": { + "description": "type_id is not meaningful for a median output_type. The property is primarily used to determine whether median is a required or optional output type through properties required and optional. If median is a required output type, the required property must be an array containing the single string element 'NA' and the optional property must be set to null. If median is an optional output type, the optional property must be an array containing the single string element 'NA' and the required property must be set to null", + "examples": [{ + "required": [ + "NA" + ], + "optional": null + }, + { + "required": null, + "optional": [ + "NA" + ] + } + ], + "type": "object", + "oneOf": [{ + "properties": { + "required": { + "description": "When median is required, property set to single element 'NA' array", + "type": "array", + "items": { + "const": "NA", + "maxItems": 1 + } + }, + "optional": { + "description": "When median is required, property set to null", + "type": "null" + } + } + }, + { + "properties": { + "required": { + "description": "When median is optional, property set to null", + "type": "null" + }, + "optional": { + "description": "When median is optional, property set to single element 'NA' array", + "type": "array", + "items": { + "const": "NA", + "maxItems": 1 + } + } + } + } + ], + "required": [ + "required", + "optional" + ] + }, + "value": { + "type": "object", + "description": "Object defining the characteristics of valid median values", + "examples": [{ + "type": "double", + "minimum": 0 + }], + "properties": { + "type": { + "description": "Data type of median values", + "type": "string", + "enum": [ + "double", + "integer" + ] + }, + "minimum": { + "description": "The minimum inclusive valid median value", + "type": [ + "number", + "integer" + ] + }, + "maximum": { + "description": "the maximum inclusive valid median value", + "type": [ + "number", + "integer" + ] + } + }, + "required": [ + "type" + ] + } + }, + "required": [ + "type_id", + "value" + ] + }, + "quantile": { + "description": "Object defining the quantiles of the predictive distribution output type.", + "type": "object", + "properties": { + "type_id": { + "description": "Object containing required and optional arrays defining the probability levels at which quantiles of the predictive distribution will be recorded.", + "examples": [{ + "required": [ + 0.25, + 0.5, + 0.75 + ], + "optional": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.6, + 0.7, + 0.8, + 0.9 + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of unique probability levels between 0 and 1 that must be present for submission to be valid. Can be null if no probability levels are required and all valid probability levels are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "optional": { + "description": "Array of valid but not required unique probability levels. Can be null if all probability levels are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "value": { + "type": "object", + "description": "Object defining the characteristics of valid quantiles of the predictive distribution at a given probability level.", + "properties": { + "type": { + "description": "Data type of quantile values.", + "examples": [ + "double" + ], + "type": "string", + "enum": [ + "double", + "integer" + ] + }, + "minimum": { + "description": "The minimum inclusive valid quantile value (optional).", + "examples": [ + 0 + ], + "type": [ + "number", + "integer" + ] + }, + "maximum": { + "description": "The maximum inclusive valid quantile value (optional).", + "type": [ + "number", + "integer" + ] + } + }, + "required": [ + "type" + ] + } + }, + "required": [ + "type_id", + "value" + ] + }, + "cdf": { + "description": "Object defining the cumulative distribution function of the predictive distribution output type.", + "type": "object", + "properties": { + "type_id": { + "description": "Object containing required and optional arrays defining possible values of the target variable at which values of the cumulative distribution function of the predictive distribution will be recorded.", + "examples": [{ + "required": [ + 10.0, + 20.0 + ], + "optional": null + }, + { + "required": [ + "EW202240", + "EW202241", + "EW202242", + "EW202243", + "EW202244", + "EW202245", + "EW202246", + "EW202247" + ], + "optional": null + } + ], + "type": "object", + "properties": { + "required": { + "description": "Array of unique target values that must be present for submission to be valid. Can be null if no target values are required and all valid target values are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "oneOf": [{ + "type": [ + "number", + "integer" + ], + "minimum": 0 + }, + { + "type": "string", + "pattern": "^EW[0-9]{6}", + "minLength": 8, + "maxLength": 8 + } + ] + } + }, + "optional": { + "description": "Array of valid but not required unique target values. Can be null if all target values are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "oneOf": [{ + "type": [ + "number", + "integer" + ], + "minimum": 0 + }, + { + "type": "string", + "pattern": "^EW[0-9]{6}", + "minLength": 8, + "maxLength": 8 + } + ] + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "value": { + "type": "object", + "description": "Object defining the characteristics of valid values of the cumulative distribution function at a given target value.", + "properties": { + "type": { + "description": "Data type of cumulative distribution function values.", + "examples": [ + "double" + ], + "const": "double" + }, + "minimum": { + "description": "The minimum inclusive valid cumulative distribution function value. Must be 0.", + "const": 0 + }, + "maximum": { + "description": "The maximum inclusive valid cumulative distribution function value. Must be 1.", + "const": 1 + } + }, + "required": [ + "type", + "minimum", + "maximum" + ] + } + }, + "required": [ + "type_id", + "value" + ] + }, + "pmf": { + "description": "Object defining a probability mass function for a discrete variable output type. Includes nominal, binary and ordinal variable types.", + "type": "object", + "properties": { + "type_id": { + "description": "Object containing required and optional arrays specifying valid categories of a discrete variable.", + "examples": [{ + "required": null, + "optional": [ + "low", + "moderate", + "high", + "extreme" + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of unique categories of a discrete variable that must be present for submission to be valid. Can be null if no categories are required and all valid categories are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "optional": { + "description": "Array of valid but not required unique categories of a discrete variable. Can be null if all categories are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "value": { + "type": "object", + "description": "Object defining valid values of the probability mass function of the predictive distribution for a given category of a discrete outcome variable.", + "examples": [{ + "type": "double", + "minimum": 0, + "maximum": 1 + }], + "properties": { + "type": { + "description": "Data type of the probability mass function values.", + "const": "double" + }, + "minimum": { + "description": "The minimum inclusive valid probability mass function value. Must be 0.", + "const": 0 + }, + "maximum": { + "description": "The maximum inclusive valid probability mass function value. Must be 1.", + "const": 1 + } + }, + "required": [ + "type", + "minimum", + "maximum" + ] + } + }, + "required": [ + "type_id", + "value" + ] + }, + "sample": { + "description": "Object defining a sample output type.", + "type": "object", + "properties": { + "type_id": { + "description": "Object containing required and optional arrays specifying valid sample values.", + "examples": [{ + "required": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "optional": [ + 11, + 12, + 13, + 14, + 15 + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of unique sample indexes that must be present for submission to be valid. Can be null if no sample indexes are required and all valid sample indexes are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "optional": { + "description": "Array of valid but not required unique sample indexes. Can be null if all sample indexes are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "value": { + "type": "object", + "description": "Object defining valid values of samples from the predictive distribution for a given sample index. Depending on the Hub specification, samples with the same sample index (specified by the type_id) may be assumed to correspond to a joint distribution across multiple levels of the task id variables. See Hub documentation for details.", + "properties": { + "type": { + "description": "Data type of sample value from the predictive distribution.", + "examples": [{ + "type": "double" + }], + "type": "string", + "enum": [ + "double", + "integer" + ] + }, + "minimum": { + "description": "The minimum inclusive valid sample value from the predictive distribution" + }, + "maximum": { + "description": "The maximum inclusive valid sample value from the predictive distribution" + } + }, + "required": [ + "type" + ] + } + }, + "required": [ + "type_id", + "value" + ] + } + }, + "additionalProperties": false + }, + "target_metadata": { + "description": "Array of objects containing metadata about each unique target, one object for each unique target value.", + "type": "array", + "items": { + "type": "object", + "description": "Object containg metadata about a single unique target.", + "properties": { + "target_id": { + "description": "Short description that uniquely identifies the target.", + "examples": [ + "inc hosp", + "peak week hosp" + ], + "type": "string", + "maxLength": 30 + }, + "target_name": { + "description": "A longer human readable target description that could be used, for example, as a visualisation axis label.", + "examples": [ + "Weekly incident influenza hospitalizations", + "Peak week for incident influenza hospitalizations" + ], + "type": "string", + "maxLength": 100 + }, + "target_units": { + "description": "Unit of observation of the target.", + "examples": [ + "rate per 100,000 population", + "count", + "date" + ], + "type": "string", + "maxLength": 100 + }, + "target_keys": { + "description": "Should be either null, in the case where the target is not specified as a task_id and is specified solely through the target_id target_metadata property or an object with one or more properties, the names of which match task_id variable(s) named within the same model_tasks object. Each property should have one specified value. Each value, or the combination of values if multiple keys are specified, define a single target value.", + "examples": [{ + "target": "inc hosp" + }, + { + "target": "peak week hosp" + }, + { + "target_variable": "hosp", + "target_outcome": "inc" + }, + { + "target_variable": "case", + "target_outcome": "peak week" + }, + null + ], + "type": [ + "object", + "null" + ] + }, + "description": { + "description": "a verbose description of the target that might include information such as the target_measure above, or definitions of a 'rate' or similar.", + "type": "string" + }, + "target_type": { + "description": "Target statistical data type", + "examples": [ + "discrete", + "ordinal" + ], + "type": "string", + "enum": [ + "continuous", + "discrete", + "date", + "binary", + "nominal", + "ordinal", + "compositional" + ] + }, + "is_step_ahead": { + "description": "Whether the target is part of a sequence of values", + "examples": [ + true, + false + ], + "type": "boolean" + }, + "time_unit": { + "description": " if is_step_ahead is true, then this is required and defines the unit of time steps. if is_step_ahead is false, then this should be left out and/or will be ignored if present.", + "examples": [ + "week" + ], + "type": "string", + "enum": [ + "day", + "week", + "month" + ] + } + }, + "required": [ + "target_id", + "target_name", + "target_units", + "target_type", + "target_keys", + "is_step_ahead" + ], + "additionalProperties": false, + "if": { + "properties": { + "is_step_ahead": { + "const": true + } + } + }, + "then": { + "required": [ + "time_unit" + ] + } + } + } + }, + "required": [ + "task_ids", + "output_type", + "target_metadata" + ] + } + }, + "submissions_due": { + "description": "Object defining the dates by which model forecasts must be submitted to the hub.", + "examples": [{ + "start": "2022-06-07", + "end": "2022-07-20" + }, + { + "relative_to": "origin_date", + "start": -4, + "end": 2 + } + ], + "type": "object", + "oneOf": [{ + "properties": { + "relative_to": { + "description": "Name of task id variable in relation to which submission start and end dates are calculated.", + "type": "string" + }, + "start": { + "description": "Difference in days between start and origin date.", + "type": "integer" + }, + "end": { + "description": "Difference in days between end and origin date.", + "type": "integer" + } + }, + "required": [ + "relative_to", + "start", + "end" + ] + }, + { + "properties": { + "start": { + "description": "Submission start date.", + "type": "string", + "format": "date" + }, + "end": { + "description": "Submission end date.", + "type": "string", + "format": "date" + } + }, + "required": [ + "start", + "end" + ] + } + ], + "required": [ + "start", + "end" + ] + }, + "last_data_date": { + "description": "The last date with recorded data in the data set used as input to a model.", + "examples": [ + "2022-07-18" + ], + "type": "string", + "format": "date" + }, + "file_format": { + "description": "Accepted file formats of model output files for the round. Overrides the file formats provided in admin.json.", + "examples": [ + ["arrow", "parquet"], + ["csv"] + ], + "type": "array", + "items": { + "type": "string", + "enum": [ + "csv", + "parquet", + "arrow" + ] + } + } + }, + "required": [ + "round_id_from_variable", + "round_id", + "model_tasks", + "submissions_due" + ], + "additionalProperties": false + } + } + }, + "required": [ + "rounds", + "schema_version" + ], + "additionalProperties": false +} From c10c704830a2ce8f422ce3a19221d6967941d5d3 Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Fri, 7 Jul 2023 12:26:43 +0300 Subject: [PATCH 03/15] Add schema for task ID additional properties. Resolves #54 --- v1.0.1/tasks-schema.json | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/v1.0.1/tasks-schema.json b/v1.0.1/tasks-schema.json index 920805b..ea4f655 100644 --- a/v1.0.1/tasks-schema.json +++ b/v1.0.1/tasks-schema.json @@ -388,6 +388,7 @@ } }, "optional": { + "description": "Array of valid but not required unique age group identifiers. Can be null if all age group are required and are specified in the required property.", "type": [ "array", "null" @@ -404,7 +405,32 @@ ] } }, - "additionalProperties": true + "additionalProperties": { + "type": "object", + "description": "An object containing arrays of required and optional unique values for a custom Task ID", + "properties": { + "required": { + "description": "Array of custom Task ID unique values that must be present for submission to be valid. Can be null if no values are required and all valid values are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true + }, + "optional": { + "description": "Array of valid but not required unique custom Task ID values. Can be null if all values are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true + } + }, + "required": [ + "required", + "optional" + ] + } }, "output_type": { "type": "object", From 28589214b7a30211b85e954360f0fe57e1fda76e Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Wed, 12 Jul 2023 10:54:35 +0300 Subject: [PATCH 04/15] Create draft v2.0.0 from unpublished v1.0.1 --- {v1.0.1 => v2.0.0}/admin-schema.json | 2 +- {v1.0.1 => v2.0.0}/tasks-schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename {v1.0.1 => v2.0.0}/admin-schema.json (99%) rename {v1.0.1 => v2.0.0}/tasks-schema.json (99%) diff --git a/v1.0.1/admin-schema.json b/v2.0.0/admin-schema.json similarity index 99% rename from v1.0.1/admin-schema.json rename to v2.0.0/admin-schema.json index 20edf5c..3e92cca 100644 --- a/v1.0.1/admin-schema.json +++ b/v2.0.0/admin-schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v1.0.1/admin-schema.json", + "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/admin-schema.json", "title": "Schema for Modeling Hub administrative settings", "description": "This JSON file provides a schema for modeling hub administrative information.", "type": "object", diff --git a/v1.0.1/tasks-schema.json b/v2.0.0/tasks-schema.json similarity index 99% rename from v1.0.1/tasks-schema.json rename to v2.0.0/tasks-schema.json index ea4f655..80675e0 100644 --- a/v1.0.1/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v1.0.1/tasks-schema.json", + "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/tasks-schema.json", "title": "Schema for Modeling Hub model task definitions", "description": "This is the schema of the tasks.json configuration file that defines the tasks within a modeling hub.", "type": "object", From 2df72a5f20b5e6b076156201d339bd39de121d2d Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Wed, 12 Jul 2023 11:00:27 +0300 Subject: [PATCH 05/15] Change type_id property to output_type_id. Resolves #55 --- v2.0.0/tasks-schema.json | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 80675e0..51a2f52 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -434,14 +434,14 @@ }, "output_type": { "type": "object", - "description": "Object defining valid model output types for a given modeling task. The name of each property corresponds to valid values in column 'type' while the 'type_id' property of each output type defines the valid values of the 'type_id' column and the 'value' property defines the valid values of the 'value' column for a given output type.", + "description": "Object defining valid model output types for a given modeling task. The name of each property corresponds to valid values in column 'output_type' while the 'output_type_id' property of each output type defines the valid values of the 'output_type_id' column and the 'value' property defines the valid values of the 'value' column for a given output type.", "properties": { "mean": { "type": "object", "description": "Object defining the mean of the predictive distribution output type.", "properties": { - "type_id": { - "description": "type_id is not meaningful for a mean output_type. The property is primarily used to determine whether mean is a required or optional output type through properties required and optional. If mean is a required output type, the required property must be an array containing the single string element 'NA' and the optional property must be set to null. If mean is an optional output type, the optional property must be an array containing the single string element 'NA' and the required property must be set to null", + "output_type_id": { + "description": "output_type_id is not meaningful for a mean output_type. The property is primarily used to determine whether mean is a required or optional output type through properties required and optional. If mean is a required output type, the required property must be an array containing the single string element 'NA' and the optional property must be set to null. If mean is an optional output type, the optional property must be an array containing the single string element 'NA' and the required property must be set to null", "examples": [{ "required": [ "NA" @@ -531,7 +531,7 @@ } }, "required": [ - "type_id", + "output_type_id", "value" ] }, @@ -539,8 +539,8 @@ "type": "object", "description": "Object defining the median of the predictive distribution output type", "properties": { - "type_id": { - "description": "type_id is not meaningful for a median output_type. The property is primarily used to determine whether median is a required or optional output type through properties required and optional. If median is a required output type, the required property must be an array containing the single string element 'NA' and the optional property must be set to null. If median is an optional output type, the optional property must be an array containing the single string element 'NA' and the required property must be set to null", + "output_type_id": { + "description": "output_type_id is not meaningful for a median output_type. The property is primarily used to determine whether median is a required or optional output type through properties required and optional. If median is a required output type, the required property must be an array containing the single string element 'NA' and the optional property must be set to null. If median is an optional output type, the optional property must be an array containing the single string element 'NA' and the required property must be set to null", "examples": [{ "required": [ "NA" @@ -630,7 +630,7 @@ } }, "required": [ - "type_id", + "output_type_id", "value" ] }, @@ -638,7 +638,7 @@ "description": "Object defining the quantiles of the predictive distribution output type.", "type": "object", "properties": { - "type_id": { + "output_type_id": { "description": "Object containing required and optional arrays defining the probability levels at which quantiles of the predictive distribution will be recorded.", "examples": [{ "required": [ @@ -730,7 +730,7 @@ } }, "required": [ - "type_id", + "output_type_id", "value" ] }, @@ -738,7 +738,7 @@ "description": "Object defining the cumulative distribution function of the predictive distribution output type.", "type": "object", "properties": { - "type_id": { + "output_type_id": { "description": "Object containing required and optional arrays defining possible values of the target variable at which values of the cumulative distribution function of the predictive distribution will be recorded.", "examples": [{ "required": [ @@ -845,7 +845,7 @@ } }, "required": [ - "type_id", + "output_type_id", "value" ] }, @@ -853,7 +853,7 @@ "description": "Object defining a probability mass function for a discrete variable output type. Includes nominal, binary and ordinal variable types.", "type": "object", "properties": { - "type_id": { + "output_type_id": { "description": "Object containing required and optional arrays specifying valid categories of a discrete variable.", "examples": [{ "required": null, @@ -924,7 +924,7 @@ } }, "required": [ - "type_id", + "output_type_id", "value" ] }, @@ -932,7 +932,7 @@ "description": "Object defining a sample output type.", "type": "object", "properties": { - "type_id": { + "output_type_id": { "description": "Object containing required and optional arrays specifying valid sample values.", "examples": [{ "required": [ @@ -989,7 +989,7 @@ }, "value": { "type": "object", - "description": "Object defining valid values of samples from the predictive distribution for a given sample index. Depending on the Hub specification, samples with the same sample index (specified by the type_id) may be assumed to correspond to a joint distribution across multiple levels of the task id variables. See Hub documentation for details.", + "description": "Object defining valid values of samples from the predictive distribution for a given sample index. Depending on the Hub specification, samples with the same sample index (specified by the output_type_id) may be assumed to correspond to a joint distribution across multiple levels of the task id variables. See Hub documentation for details.", "properties": { "type": { "description": "Data type of sample value from the predictive distribution.", @@ -1015,7 +1015,7 @@ } }, "required": [ - "type_id", + "output_type_id", "value" ] } From b562b37609429c088c17d261db7688ad1c2bb89b Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Wed, 12 Jul 2023 17:45:19 +0300 Subject: [PATCH 06/15] Add forecast_date / target_end_date. Resolves #53 --- v2.0.0/tasks-schema.json | 84 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 51a2f52..0989c28 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -90,6 +90,48 @@ "optional" ] }, + "forecast_date": { + "description": "An object containing arrays of required and optional unique forecast dates. Forecast date usually defines the starting point that can be used for calculating a target_date via the formula target_date = origin_date + horizon x time_units_per_horizon (e.g., with weekly data, target_date is calculated as origin_date + horizon x 7 days)", + "examples": [{ + "required": null, + "optional": [ + "2022-11-05", + "2022-11-12", + "2022-11-19" + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of forecast date unique identifiers that must be present for submission to be valid. Can be null if no origin dates are required and all valid forecast dates are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + }, + "optional": { + "description": "Array of valid but not required unique forecast date identifiers. Can be null if all forecast dates are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + } + }, + "required": [ + "required", + "optional" + ] + }, "scenario_id": { "description": "An object containing arrays of required and optional unique identifiers of each valid scenario.", "examples": [{ @@ -315,6 +357,48 @@ "optional" ] }, + "target_end_date": { + "description": "An object containing arrays of required and optional unique target end dates. For short-term forecasts, the target_end_date specifies the date of occurrence of the outcome of interest. For instance, if models are requested to forecast the number of hospitalizations that will occur on 2022-07-15, the target_end_date is 2022-07-15", + "examples": [{ + "required": null, + "optional": [ + "2022-11-12", + "2022-11-19", + "2022-11-26" + ] + }], + "type": "object", + "properties": { + "required": { + "description": "Array of target end date unique identifiers that must be present for submission to be valid. Can be null if no target end dates are required and all valid target end dates are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + }, + "optional": { + "description": "Array of valid but not required unique target end date identifiers. Can be null if all target end dates are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string", + "format": "date" + } + } + }, + "required": [ + "required", + "optional" + ] + }, "horizon": { "description": "An object containing arrays of required and optional unique horizons. Horizons define the difference between the target_date and the origin_date in time units specified by the hub (e.g., may be days, weeks, or months)", "examples": [{ From 84dce1a89afd89041932628b89bb2e68ea585620 Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Wed, 12 Jul 2023 18:02:44 +0300 Subject: [PATCH 07/15] Update NEWS with v2.0.0 changes --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index edbbef7..acf1846 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +# v2.0.0 + +* Major breaking change: Output type property `type_id` renamed to `output_type_id` for consistency in with changes in model output data. +* Added `uniqueItem` checks to all `optional` and `required` arrays. +* Added a schema to `task_ids` `additionalProperty` property instead of just setting it to `true`. This schema ensures that any custom task IDs added by hub administrators at least checked for being objects and having `optional` and `required` properties that are either arrays or null. +* Added standard task IDs `forecast_date` (equivalent to `origin_date`), `target_end_date` (equivalent to `target_date`). + # v1.0.0 * Major breaking change: `categorical` output type renamed to `pmf` for consistency in terminology. From 73b964ed73c5adc20a7a304fc59d89173c336032 Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Wed, 12 Jul 2023 18:35:26 +0300 Subject: [PATCH 08/15] correct carry over mention of origin_date in description --- v2.0.0/tasks-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 0989c28..2bd0875 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -91,7 +91,7 @@ ] }, "forecast_date": { - "description": "An object containing arrays of required and optional unique forecast dates. Forecast date usually defines the starting point that can be used for calculating a target_date via the formula target_date = origin_date + horizon x time_units_per_horizon (e.g., with weekly data, target_date is calculated as origin_date + horizon x 7 days)", + "description": "An object containing arrays of required and optional unique forecast dates. Forecast date usually defines the starting point that can be used for calculating a target_date via the formula target_date = forecast_date + horizon x time_units_per_horizon (e.g., with weekly data, target_date is calculated as forecast_date + horizon x 7 days)", "examples": [{ "required": null, "optional": [ From da2a512c085affa74c62ca8d8541cf3a6ab1279f Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Thu, 13 Jul 2023 10:18:52 +0300 Subject: [PATCH 09/15] Update forecast_date description Co-authored-by: Evan Ray --- v2.0.0/tasks-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 2bd0875..645cfb7 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -91,7 +91,7 @@ ] }, "forecast_date": { - "description": "An object containing arrays of required and optional unique forecast dates. Forecast date usually defines the starting point that can be used for calculating a target_date via the formula target_date = forecast_date + horizon x time_units_per_horizon (e.g., with weekly data, target_date is calculated as forecast_date + horizon x 7 days)", + "description": "An object containing arrays of required and optional unique forecast dates. Forecast date usually defines the date that a model is run to produce a forecast.", "examples": [{ "required": null, "optional": [ From e2a6d7f94e425cc325e30576b805d21c9d84bf3f Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Thu, 13 Jul 2023 10:21:48 +0300 Subject: [PATCH 10/15] Use http instead of https in schema property. Resolves #44. Resolves #42 --- v2.0.0/admin-schema.json | 2 +- v2.0.0/tasks-schema.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2.0.0/admin-schema.json b/v2.0.0/admin-schema.json index 3e92cca..849eb04 100644 --- a/v2.0.0/admin-schema.json +++ b/v2.0.0/admin-schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/admin-schema.json", "title": "Schema for Modeling Hub administrative settings", "description": "This JSON file provides a schema for modeling hub administrative information.", diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 645cfb7..8660044 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/tasks-schema.json", "title": "Schema for Modeling Hub model task definitions", "description": "This is the schema of the tasks.json configuration file that defines the tasks within a modeling hub.", @@ -103,7 +103,7 @@ "type": "object", "properties": { "required": { - "description": "Array of forecast date unique identifiers that must be present for submission to be valid. Can be null if no origin dates are required and all valid forecast dates are specified in the optional property.", + "description": "Array of forecast date unique identifiers that must be present for submission to be valid. Can be null if no forecast dates are required and all valid forecast dates are specified in the optional property.", "type": [ "array", "null" From 56d98520b8d1341b47df83bcdec19ebe26f4884e Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Thu, 13 Jul 2023 10:28:18 +0300 Subject: [PATCH 11/15] re-introduce https --- v2.0.0/admin-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2.0.0/admin-schema.json b/v2.0.0/admin-schema.json index 849eb04..3e92cca 100644 --- a/v2.0.0/admin-schema.json +++ b/v2.0.0/admin-schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/admin-schema.json", "title": "Schema for Modeling Hub administrative settings", "description": "This JSON file provides a schema for modeling hub administrative information.", From 9d1444e9f22fc61f9cb40905ec1ae2f1c3dc743f Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Thu, 13 Jul 2023 10:30:47 +0300 Subject: [PATCH 12/15] re-introduce https --- v2.0.0/tasks-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 8660044..36e9dd4 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/tasks-schema.json", "title": "Schema for Modeling Hub model task definitions", "description": "This is the schema of the tasks.json configuration file that defines the tasks within a modeling hub.", From 12a654f8b28f962a6978819c5fc8e6c9a9817fd8 Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Thu, 13 Jul 2023 11:05:16 +0300 Subject: [PATCH 13/15] Add target_variable & target_outcome std task IDs. Resolves #29 --- v2.0.0/tasks-schema.json | 80 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 36e9dd4..27be334 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -278,7 +278,7 @@ ] }, "target": { - "description": "An object containing arrays of required and optional unique identifiers for each valid target.", + "description": "An object containing arrays of required and optional unique identifiers for each valid target. Usually represents a single task ID target key variable.", "type": "object", "examples": [{ "required": null, @@ -315,6 +315,84 @@ "optional" ] }, + "target_variable": { + "description": "An object containing arrays of required and optional unique identifiers for each valid target variable. Usually forms part of a pair of task ID target key variables (along with target_outcome) which combine to define individual targets.", + "type": "object", + "examples": [{ + "required": null, + "optional": ["hosp", "death", "case"] + }, { + "required": ["hosp"], + "optional": ["case"] + }], + "properties": { + "required": { + "description": "Array of target variable unique identifiers that must be present for submission to be valid. Can be null if no target variables are required and all valid target variables are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "optional": { + "description": "Array of valid but not required unique target variable identifiers. Can be null if all target variables are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + } + }, + "required": [ + "required", + "optional" + ] + }, + "target_outcome": { + "description": "An object containing arrays of required and optional unique identifiers for each valid target outcome. Usually forms part of a pair of task ID target key variables (along with target_variable) which combine to define individual targets.", + "type": "object", + "examples": [{ + "required": ["inc"], + "optional": null + }, { + "required": ["inc"], + "optional": ["cum"] + }], + "properties": { + "required": { + "description": "Array of target outcome unique identifiers that must be present for submission to be valid. Can be null if no target outcomes are required and all valid target outcomes are specified in the optional property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "optional": { + "description": "Array of valid but not required unique target outcome identifiers. Can be null if all target outcomes are required and are specified in the required property.", + "type": [ + "array", + "null" + ], + "uniqueItems": true, + "items": { + "type": "string" + } + } + }, + "required": [ + "required", + "optional" + ] + }, "target_date": { "description": "An object containing arrays of required and optional unique target dates. For short-term forecasts, the target_date specifies the date of occurrence of the outcome of interest. For instance, if models are requested to forecast the number of hospitalizations that will occur on 2022-07-15, the target_date is 2022-07-15", "examples": [{ From 611962ee707a7a3907641ba4736c8281c5e18d0b Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Thu, 13 Jul 2023 11:11:18 +0300 Subject: [PATCH 14/15] add better examples to target task ID --- v2.0.0/tasks-schema.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/v2.0.0/tasks-schema.json b/v2.0.0/tasks-schema.json index 27be334..8b452bb 100644 --- a/v2.0.0/tasks-schema.json +++ b/v2.0.0/tasks-schema.json @@ -283,8 +283,13 @@ "examples": [{ "required": null, "optional": [ - "weekly rate" + "inc hosp", + "inc case", + "inc death" ] + }, { + "required": ["peak week inc hosp"], + "optional": null }], "properties": { "required": { From b638b7e1d5e71dc7bf998448a254355772fd2085 Mon Sep 17 00:00:00 2001 From: Anna Krystalli Date: Thu, 13 Jul 2023 11:17:24 +0300 Subject: [PATCH 15/15] add note about target_variable & target_outcome to news --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index acf1846..da093b7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,7 +3,7 @@ * Major breaking change: Output type property `type_id` renamed to `output_type_id` for consistency in with changes in model output data. * Added `uniqueItem` checks to all `optional` and `required` arrays. * Added a schema to `task_ids` `additionalProperty` property instead of just setting it to `true`. This schema ensures that any custom task IDs added by hub administrators at least checked for being objects and having `optional` and `required` properties that are either arrays or null. -* Added standard task IDs `forecast_date` (equivalent to `origin_date`), `target_end_date` (equivalent to `target_date`). +* Added standard task IDs `forecast_date` (equivalent to `origin_date`), `target_end_date` (equivalent to `target_date`) and `target_variable` & `target_outcome` which can be used to split targets across two task IDs. # v1.0.0