Update json schema creation to v0.3.0

Author: m-mohr

README.md

@@ -89,7 +89,7 @@ you can for example run:
 - GeoJSON Features (with indentation and max. 100 features):
   `fiboa create-geojson example.parquet -o dest-folder -n 100 -i 2 -f`
 
-Check `fiboa create-geoparquet --help` for more details.
+Check `fiboa create-geojson --help` for more details.
 
 ### Inspect fiboa GeoParquet file
 

fiboa_cli/__init__.py

@@ -6,7 +6,12 @@
 import click
 import pandas as pd
 
-from .const import COMPRESSION_METHODS, CORE_COLUMNS
+from .const import (
+    COMPRESSION_METHODS,
+    CORE_COLUMNS,
+    FIBOA_GEOJSON_DATATYPES_SCHEMA,
+    FIBOA_SPECIFICAION_SCHEMA,
+)
 from .convert import convert as convert_
 from .convert import list_all_converter_ids, list_all_converters
 from .create_geojson import create_geojson as create_geojson_
@@ -95,13 +100,6 @@ def describe(file, json, num=10, column=[]):
         value, ["parquet", "geoparquet", "json", "geojson"]
     ),
 )
-@click.option(
-    "--schemas",
-    "-s",
-    multiple=True,
-    callback=lambda ctx, param, value: check_ext_schema_for_cli(value, allow_none=False),
-    help="Maps a remote fiboa schema URL to a local file. First the URL, then the local file path. Separated with a comma character. Example: https://example.com/schema.yaml,/path/to/schema.yaml",
-)
 @click.option(
     "--data",
     "-d",
@@ -110,6 +108,13 @@ def describe(file, json, num=10, column=[]):
     help="EXPERIMENTAL: Validate the data in the GeoParquet file. Enabling this might be slow or exceed memory. Default is False.",
     default=False,
 )
+@click.option(
+    "--schemas",
+    "-s",
+    multiple=True,
+    callback=lambda ctx, param, value: check_ext_schema_for_cli(value, allow_none=False),
+    help="Maps a remote fiboa schema URL to a local file. First the URL, then the local file path. Separated with a comma character. Example: https://example.com/schema.yaml,/path/to/schema.yaml",
+)
 @click.option(
     "--timer",
     is_flag=True,
@@ -118,7 +123,7 @@ def describe(file, json, num=10, column=[]):
     default=False,
     hidden=True,
 )
-def validate(files, schemas, data, timer):
+def validate(files, data, schemas, timer):
     """
     Validates a fiboa GeoParquet or GeoJSON file.
     """
@@ -204,51 +209,21 @@ def validate_schema(files, metaschema):
     "--out", "-o", type=click.Path(exists=False), help="Path to write the file to.", required=True
 )
 @click.option(
-    "--collection",
-    "-c",
-    callback=valid_file_for_cli,
-    help="Points to the Collection that defines the fiboa version and extensions. Only applies if not provided in the GeoJSON file (embedded or as link).",
-    default=None,
-)
-@click.option(
-    "--schema",
+    "--schemas",
     "-s",
-    type=click.Path(exists=True),
-    help="fiboa Schema to work against. If not provided, uses the fiboa version from the collection to load the schema for the released version.",
-)
-@click.option(
-    "--ext-schema",
-    "-e",
     multiple=True,
-    callback=lambda ctx, param, value: check_ext_schema_for_cli(value, allow_none=True),
-    help="Applicable fiboa extensions as URLs. Can map a remote fiboa extension schema url to a local file by adding a local file path, separated by a comma. Example: https://example.com/schema.json,/path/to/schema.json",
-)
-@click.option(
-    "--fiboa-version",
-    "-f",
-    type=click.STRING,
-    help="The applicable fiboa version if no collection is provided.",
-    show_default=True,
-    default=fiboa_version_,
+    callback=lambda ctx, param, value: check_ext_schema_for_cli(value, allow_none=False),
+    help="Maps a remote fiboa schema URL to a local file. First the URL, then the local file path. Separated with a comma character. Example: https://example.com/schema.yaml,/path/to/schema.yaml",
 )
-def create_geoparquet(files, out, collection, schema, ext_schema, fiboa_version):
+def create_geoparquet(files, out, schemas):
     """
     Create a fiboa GeoParquet file from GeoJSON file(s).
-
-    The collection metadata has the following priority order:
-    1. Read from the last GeoJSON file/feature (embedded 'fiboa' property)
-    1. Read from the last GeoJSON file/feature (link with relation type 'collection')
-    2. Read from the collection parameter
-    3. Use fiboa_version and extension_schemas parameters
     """
     log(f"fiboa CLI {__version__} - Create GeoParquet\n", "success")
     config = {
         "files": files,
         "out": out,
-        "schema": schema,
-        "collection": collection,
-        "extension_schemas": ext_schema,
-        "fiboa_version": fiboa_version,
+        "schemas": schemas,
     }
     try:
         create_geoparquet_(config)
@@ -316,7 +291,18 @@ def create_geojson(file, out, features=False, num=None, indent=None):
     "-s",
     type=click.STRING,
     callback=valid_file_for_cli,
-    help="fiboa Schema to create the JSON Schema for. Can be a local file or a URL. If not provided, uses the fiboa version to load the schema for the released version.",
+    help=f"fiboa schema to create the JSON Schema for. Can be a local file or a URL. If not provided, loads the schema for fiboa version {fiboa_version_}.",
+    show_default=True,
+    default=FIBOA_SPECIFICAION_SCHEMA.format(version=fiboa_version_),
+)
+@click.option(
+    "--datatypes",
+    "-d",
+    type=click.STRING,
+    callback=valid_file_for_cli,
+    help=f"Schema for the fiboa GeoJSON datatypes. Can be a local file or a URL. If not provided, loads the GeoJSON datatypes for fiboa version {fiboa_version_}.",
+    show_default=True,
+    default=FIBOA_GEOJSON_DATATYPES_SCHEMA.format(version=fiboa_version_),
 )
 @click.option(
     "--out",
@@ -325,33 +311,21 @@ def create_geojson(file, out, features=False, num=None, indent=None):
     help="Path to write the file to. If not provided, prints the file to the STDOUT.",
     default=None,
 )
-@click.option(
-    "--fiboa-version",
-    "-f",
-    type=click.STRING,
-    help="The fiboa version to validate against.",
-    show_default=True,
-    default=fiboa_version_,
-)
 @click.option(
     "--id",
     "-i",
     "id_",
     type=click.STRING,
     help="The JSON Schema $id to use for the schema. If not provided, the $id will be omitted.",
+    default=None,
 )
-def jsonschema(schema, out, fiboa_version, id_):
+def jsonschema(schema, datatypes, out, id_):
     """
     Create a JSON Schema for a fiboa Schema
     """
     log(f"fiboa CLI {__version__} - Create JSON Schema\n", "success")
-    config = {
-        "schema": schema,
-        "fiboa_version": fiboa_version,
-        "id": id_,
-    }
     try:
-        schema = jsonschema_(config)
+        schema = jsonschema_(schema, datatypes, id_)
         if out:
             with open(out, "w", encoding="utf-8") as f:
                 json.dump(schema, f, indent=2)

fiboa_cli/const.py

@@ -2,6 +2,12 @@
 
 SUPPORTED_PROTOCOLS = ["http", "https", "s3", "gs"]
 
+FIBOA_SPECIFICAION_PATTERN = r"https://fiboa.github.io/specification/v([^/]+)/schema.yaml"
+FIBOA_SPECIFICAION_SCHEMA = "https://fiboa.github.io/specification/v{version}/schema.yaml"
+FIBOA_GEOJSON_DATATYPES_SCHEMA = (
+    "https://fiboa.github.io/specification/v{version}/geojson/datatypes.json"
+)
+
 GEOPARQUET_SCHEMA = "https://geoparquet.org/releases/v{version}/schema.json"
 STAC_TABLE_EXTENSION = "https://stac-extensions.github.io/table/v1.2.0/schema.json"
 

fiboa_cli/create_geojson.py

@@ -75,6 +75,24 @@ def fix_geojson(obj):
         obj["bbox"] = [bbox["xmin"], bbox["ymin"], bbox["xmax"], bbox["ymax"]]
         del obj["properties"]["bbox"]
 
+    # Remove null values
+    obj["properties"] = fix_omit_nulled_properties(obj["properties"])
+
+    return obj
+
+
+def fix_omit_nulled_properties(obj):
+    for key in obj.keys():
+        if obj[key] is None:
+            del obj[key]
+        elif isinstance(obj[key], dict):
+            obj[key] = fix_omit_nulled_properties(obj[key])
+        elif isinstance(obj[key], list):
+            for i, item in enumerate(obj[key]):
+                if not isinstance(item, dict):
+                    continue
+                obj[key][i] = fix_omit_nulled_properties(item)
+
     return obj
 
 

fiboa_cli/create_geoparquet.py

@@ -1,7 +1,5 @@
-import os
-
 from .parquet import create_parquet
-from .util import get_collection, load_file, log
+from .util import collection_from_featurecollection, load_file, log
 
 
 def create_geoparquet(config):
@@ -11,40 +9,21 @@ def create_geoparquet(config):
     features = []
     geojson = {}
     file = None
+    collection = {}
     files = config.get("files")
     for file in files:
         geojson = load_file(file)
         if geojson["type"] == "Feature":
             features.append(geojson)
         elif geojson["type"] == "FeatureCollection":
             features += geojson["features"]
+            collection = collection_from_featurecollection(geojson)
         else:
             log(f"{file}: Skipped - Unsupported GeoJSON type, must be Feature or FeatureCollection")
 
     if len(features) == 0:
         raise Exception("No valid features provided as input files")
 
-    # Add a STAC collection to the fiboa property to the Parquet metadata
-    # Note: for features this loads the collection of the last feature only
-    # if not provided specifically via collection parameter
-    collection = get_collection(geojson, config.get("collection"), file)
-
-    if collection is None:
-        # No collection found, create a default collection based on parameters
-        version = config.get("fiboa_version")
-        collection = {
-            "fiboa_version": version,
-            "fiboa_extensions": list(config.get("extension_schemas", {}).keys()),
-        }
-
-    # add a default id based on the output filename
-    if "id" not in collection or not collection["id"]:
-        collection["id"] = os.path.basename(output_file)
-
-    # Make the fiboa_version consistent with the collection
-    if "fiboa_version" in collection:
-        config["fiboa_version"] = collection["fiboa_version"]
-
     # Get a list of the properties/columns (without duplicates)
     columns = set(["id", "geometry"])
     for feature in features:

fiboa_cli/describe.py

@@ -36,7 +36,7 @@ def describe(file, display_json=False, num=10, columns=None):
         if "fiboa_extensions" in collection and isinstance(collection["fiboa_extensions"], list):
             log_extensions(collection["fiboa_extensions"], log)
 
-        custom_schemas = collection.get("fiboa_custom_schemas", {})
+        custom_schemas = collection.get("custom_schemas", {})
         if not is_schema_empty(custom_schemas):
             log("Custom schemas: " + ", ".join(custom_schemas["properties"].keys()))
 

fiboa_cli/improve.py

@@ -90,10 +90,10 @@ def improve(
         gdf["area"] = gdf_m["area"].fillna(gdf_m.geometry.area * 0.0001)
         gdf["perimeter"] = gdf_m["perimeter"].fillna(gdf_m.geometry.length)
 
-    custom_schemas = collection.get("fiboa_custom_schemas", {})
+    custom_schemas = collection.get("custom_schemas", {})
     custom_schemas = pick_schemas(custom_schemas, columns, rename_columns)
     if not is_schema_empty(custom_schemas):
-        collection["fiboa_custom_schemas"] = custom_schemas
+        collection["custom_schemas"] = custom_schemas
 
     # Write the merged dataset to the output file
     create_parquet(