Upgrade validation to v0.3.0

m-mohr · m-mohr · commit 57070cda31c0 · 2025-03-11T13:30:47.000+01:00
diff --git a/fiboa_cli/__init__.py b/fiboa_cli/__init__.py
@@ -96,32 +96,11 @@ def describe(file, json, num=10, column=[]):
     ),
 )
 @click.option(
-    "--schema",
+    "--schemas",
     "-s",
-    type=click.STRING,
-    callback=valid_file_for_cli,
-    help="fiboa Schema to validate against. Can be a local file or a URL. If not provided, uses the fiboa version to load the schema for the released version.",
-)
-@click.option(
-    "--ext-schema",
-    "-e",
     multiple=True,
     callback=lambda ctx, param, value: check_ext_schema_for_cli(value, allow_none=False),
-    help="Maps a remote fiboa extension schema url to a local file. First the URL, then the local file path. Separated with a comma character. Example: https://example.com/schema.yaml,/path/to/schema.yaml",
-)
-@click.option(
-    "--fiboa-version",
-    "-f",
-    type=click.STRING,
-    help="The fiboa version to validate against. Default is the version given in the collection.",
-    default=None,
-)
-@click.option(
-    "--collection",
-    "-c",
-    type=click.Path(exists=True),
-    help="Points to the Collection that defines the fiboa version and extensions.",
-    default=None,
+    help="Maps a remote fiboa schema URL to a local file. First the URL, then the local file path. Separated with a comma character. Example: https://example.com/schema.yaml,/path/to/schema.yaml",
 )
 @click.option(
     "--data",
@@ -139,17 +118,14 @@ def describe(file, json, num=10, column=[]):
     default=False,
     hidden=True,
 )
-def validate(files, schema, ext_schema, fiboa_version, collection, data, timer):
+def validate(files, schemas, data, timer):
     """
     Validates a fiboa GeoParquet or GeoJSON file.
     """
     start = time.perf_counter()
     log(f"fiboa CLI {__version__} - Validator\n", "success")
     config = {
-        "schema": schema,
-        "extension_schemas": ext_schema,
-        "fiboa_version": fiboa_version,
-        "collection": collection,
+        "schemas": schemas,
         "data": data,
     }
 
diff --git a/fiboa_cli/const.py b/fiboa_cli/const.py
@@ -2,9 +2,6 @@
 
 SUPPORTED_PROTOCOLS = ["http", "https", "s3", "gs"]
 
-STAC_COLLECTION_SCHEMA = (
-    "http://schemas.stacspec.org/v{version}/collection-spec/json-schema/collection.json"
-)
 GEOPARQUET_SCHEMA = "https://geoparquet.org/releases/v{version}/schema.json"
 STAC_TABLE_EXTENSION = "https://stac-extensions.github.io/table/v1.2.0/schema.json"
 
diff --git a/fiboa_cli/describe.py b/fiboa_cli/describe.py
@@ -34,7 +34,7 @@ def describe(file, display_json=False, num=10, columns=None):
     if collection:
         log(f"fiboa version: {collection['fiboa_version']}")
         if "fiboa_extensions" in collection and isinstance(collection["fiboa_extensions"], list):
-            log_extensions(collection, log)
+            log_extensions(collection["fiboa_extensions"], log)
 
         custom_schemas = collection.get("fiboa_custom_schemas", {})
         if not is_schema_empty(custom_schemas):
diff --git a/fiboa_cli/util.py b/fiboa_cli/util.py
@@ -324,27 +324,27 @@ def to_iso8601(dt):
         return iso + "Z"
 
 
-def load_collection_schema(obj):
-    if "stac_version" in obj:
-        return load_file(STAC_COLLECTION_SCHEMA.format(version=obj["stac_version"]))
-    else:
-        return None
-
-
 def load_geoparquet_schema(obj):
     if "version" in obj:
         return load_file(GEOPARQUET_SCHEMA.format(version=obj["version"]))
     else:
         return None
 
 
-def log_extensions(collection, logger):
-    extensions = collection.get("fiboa_extensions", [])
-    if len(extensions) == 0:
+def get_core_version(uri):
+    match = re.match(r"https://fiboa.github.io/specification/v([^/]+)/schema.yaml", uri)
+    return match.group(1) if match else None
+
+
+def log_extensions(schemas, logger):
+    schemas = schemas.sort()
+    if len(schemas) <= 1:
         logger("fiboa extensions: none")
     else:
         logger("fiboa extensions:")
-        for extension in extensions:
+        for extension in schemas:
+            if get_core_version(extension) is not None:
+                continue
             logger(f"  - {extension}")
 
 
diff --git a/fiboa_cli/validate.py b/fiboa_cli/validate.py
@@ -1,13 +1,13 @@
 import json
+import re
 
 import pyarrow.types as pat
 
 from .jsonschema import create_jsonschema
 from .types import PA_TYPE_CHECK
 from .util import (
     create_validator,
-    get_collection,
-    load_collection_schema,
+    get_core_version,
     load_datatypes,
     load_fiboa_schema,
     load_file,
@@ -20,6 +20,7 @@
 )
 from .util import log as log_
 from .validate_data import validate_column
+from .version import fiboa_version
 
 
 def log(text: str, status="info", bullet=True):
@@ -35,91 +36,94 @@ def validate(file, config):
         return validate_parquet(file, config)
 
 
-def validate_collection(collection, config):
+def validate_schemas(schema_uris, config):
     valid = True
 
-    collection_version = collection.get("fiboa_version")
-    config_version = config.get("fiboa_version")
+    if not isinstance(schema_uris, list):
+        log("A list of schemas must be provied", "error")
+        valid = False
 
-    # Check fiboa version
-    if not isinstance(collection_version, str):
-        log("No fiboa_version string found in collection metadata", "error")
+    if len(schema_uris) == 0:
+        log("No schemas provided", "error")
         valid = False
 
-    log("fiboa version: " + config_version)
+    # Detect and check fiboa version
+    version = None
+    core_schema = None
+    for schema_uri in schema_uris:
+        version = get_core_version(schema_uri)
+        if version is not None:
+            core_schema = schema_uri
+            break
+
+    if version is None:
+        log("fiboa core schema not found in schemas, can't detect fiboa version", "error")
+        valid = False
 
-    if isinstance(collection_version, str) and collection_version != config_version:
+    if version != fiboa_version:
         log(
-            f"fiboa versions differs: Collection is {collection_version} and requested specification version is {config_version}",
+            f"fiboa versions differs: Schema reports {version} and supported version is {fiboa_version}",
             "warning",
         )
 
-    # Check STAC Collection
-    if not validate_colletion_schema(collection):
-        valid = False
+    # Check schemas (core and extensions)
+    schemas = {}
+    schema_map = config.get("schemas", {})
+    for uri in schema_uris:
+        try:
+            if uri in schema_map:
+                actual_location = schema_map[uri]
+                log(f"Redirecting {uri} to {actual_location}", "info")
+            else:
+                actual_location = uri
 
-    # Check extensions
-    extensions = {}
-    if "fiboa_extensions" in collection:
-        ext_list = collection.get("fiboa_extensions")
-        if not isinstance(ext_list, list):
-            log("fiboa_extensions must be a list", "error")
+            schemas[schema_uri] = load_file(actual_location)
+        except Exception as e:
+            log(f"Extension {uri} can't be loaded: {e}", "error")
             valid = False
-        else:
-            ext_map = config.get("extension_schemas", [])
-            for ext in ext_list:
-                try:
-                    if ext in ext_map:
-                        path = ext_map[ext]
-                        log(f"Redirecting {ext} to {path}", "info")
-                    else:
-                        path = ext
-                    extensions[ext] = load_file(path)
-                except Exception as e:
-                    log(f"Extension {ext} can't be loaded: {e}", "error")
-                    valid = False
 
-    log_extensions(collection, lambda x: log(x, "info", False))
+    log_extensions(schema_uris, lambda x: log(x, "info", False))
 
-    return valid, extensions
+    return valid, version, core_schema, schemas
 
 
 def validate_geojson(file, config):
-    valid = True
-    extensions = {}
-
     try:
         data = load_file(file)
     except Exception as error:
         log(error, "error")
         return False
 
-    collection = get_collection(data, config.get("collection"), file)
-    if collection is None:
-        log("No collection specified", "error")
-        valid = False
+    if not isinstance(data, dict):
+        log("Must be a JSON object", "error")
+        return False
 
-    if config.get("fiboa_version") is None and collection.get("fiboa_version") is not None:
-        config["fiboa_version"] = collection.get("fiboa_version")
+    schemas_uris = {}
+    collection = {}
+    feature_type = data.get("type")
+    if feature_type == "FeatureCollection":
+        collection = data.get("fiboa", {})
+        schemas_uris = collection.get("schemas", {})
+        features = data["features"]
+    elif feature_type == "Feature":
+        schemas_uris = data.get("properties", {}).get("schemas", {})
+        features = [data]
+    else:
+        log("Must be a GeoJSON Feature or FeatureCollection", "error")
+        return False
 
-    if collection is not None:
-        collection_valid, extensions = validate_collection(collection, config)
-        if not collection_valid:
-            valid = False
+    valid, version, core_schema_uri, schemas = validate_schemas(schemas_uris, config)
 
-    core_schema = load_fiboa_schema(config)
-    datatypes = load_datatypes(config["fiboa_version"])
+    core_schema = schemas[core_schema_uri]
+    datatypes = load_datatypes(version)
     schema = create_jsonschema(core_schema, datatypes)
 
     # Load extensions
     ext_errors = []
-    for ext in extensions:
+    extensions = {}
+    for ext in schemas:
         try:
-            uri = ext
-            if ext in config["extension_schemas"]:
-                uri = config["extension_schemas"][ext]
-            ext_schema = load_file(uri)
-            json_schema = create_jsonschema(ext_schema, datatypes)
+            json_schema = create_jsonschema(schemas[ext], datatypes)
             extensions[ext] = lambda obj: validate_json_schema(obj, json_schema)
         except Exception as error:
             extensions[ext] = None
@@ -129,21 +133,6 @@ def validate_geojson(file, config):
         log(error, "error")
 
     # Validate
-    if not isinstance(data, dict):
-        log("Must be a JSON object", "error")
-        return False
-
-    if data["type"] == "Feature":
-        features = [data]
-    elif data["type"] == "FeatureCollection":
-        features = data["features"]
-    elif data["type"] == "Collection":
-        # Skipping specific checks related to STAC Collection
-        return None
-    else:
-        log("Must be a GeoJSON Feature or FeatureCollection", "error")
-        return False
-
     if len(features) == 0:
         log("Must contain at least one Feature", "error")
         return False
@@ -176,41 +165,28 @@ def validate_geojson(file, config):
 
 def validate_parquet(file, config):
     parquet_schema = load_parquet_schema(file)
-    valid = True
-    extensions = {}
 
     # Validate geo metadata in Parquet header
     if b"geo" not in parquet_schema.metadata:
         log("Parquet file schema does not have 'geo' key", "error")
-        valid = False
+        return False
     else:
         geo = parse_metadata(parquet_schema, b"geo")
         if not validate_geoparquet_schema(geo):
-            valid = False
+            return False
 
     # Validate fiboa metadata in Parquet header
-    collection = {}
     if b"fiboa" not in parquet_schema.metadata:
-        log("Parquet file schema does not have a 'fiboa' key", "warning")
-        if not config.get("collection"):
-            log("No collection specified", "error")
-            valid = False
-        else:
-            collection = load_file(config.get("collection"))
-    else:
-        collection = parse_metadata(parquet_schema, b"fiboa")
+        log("Parquet file schema does not have a 'fiboa' key", "error")
+        return False
 
-    if config.get("fiboa_version") is None and collection.get("fiboa_version") is not None:
-        config["fiboa_version"] = collection.get("fiboa_version")
+    collection = parse_metadata(parquet_schema, b"fiboa")
+    schemas_uris = collection.get("schemas", {})
 
-    # Validate Collection
-    if len(collection) > 0:
-        valid_collection, extensions = validate_collection(collection, config)
-        if not valid_collection:
-            valid = False
+    valid, version, core_schema_uri, schemas = validate_schemas(schemas_uris, config)
 
-    # load the actual fiboa schema
-    fiboa_schema = load_fiboa_schema(config)
+    fiboa_schema = schemas[core_schema_uri]
+    extensions = {}
 
     # Load data if needed
     gdf = None
@@ -223,7 +199,9 @@ def validate_parquet(file, config):
 
     # Compile all properties from the schemas
     schemas = fiboa_schema
-    for ext in extensions.values():
+    for ext in schemas.values():
+        if core_schema_uri == ext:
+            continue
         schemas = merge_schemas(schemas, ext)
 
     # Add custom schemas
@@ -326,22 +304,6 @@ def validate_geometry_column(key, prop_schema, geo, valid=True):
     return valid
 
 
-# todo: use stac_validator instead of our own validation routine
-def validate_colletion_schema(obj):
-    if "stac_version" in obj:
-        try:
-            schema = load_collection_schema(obj)
-            errors = validate_json_schema(obj, schema)
-            for error in errors:
-                log(f"Collection: {error.path}: {error.message}", "error")
-
-            return len(errors) == 0
-        except Exception as e:
-            log(f"Failed to validate STAC Collection due to an internal error: {e}", "warning")
-
-    return True
-
-
 # todo: use a geoparquet validator instead of our own validation routine
 def validate_geoparquet_schema(obj):
     if "version" in obj:
diff --git a/fiboa_cli/version.py b/fiboa_cli/version.py
@@ -1,2 +1,2 @@
 __version__ = "0.10.0"
-fiboa_version = "0.2.0"
+fiboa_version = "0.3.0"

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`__version__ = "0.10.0"`
`2`		`-fiboa_version = "0.2.0"`
	`2`	`+fiboa_version = "0.3.0"`