Feature / Support FILE inputs and outputs for models (#487)

* Update metadata protos to support file type for model input / output * Update static API for defining model inputs and outputs * Fix license headers that were using the old template * Update static API for dynamic schemas and compatibility of MIS / MOS types * Update scan_model() to handle new ModelInput / ModelOutput types * Dev mode translation for file IO * Use single instance of resources with the dev mode translator * Change ordering of imports in top level API package (allow overriding metadata types for backward compatibility) * Minor log fix in dev mode translator * Update data classes to support both FILE and DATA object types * Update graph processing - use updated data classes and simplify load / save dependencies * Fix ext plugin loader tests * Fix core jobs test * Fix trac context test * Fix dev mode inference for flow outputs * Update graph build to handle file inputs and outputs * Updates in data and functions modules to support file IO * Implement get_file() / put_file() in model API and context impl * Add support for model file IO using streams * Improve output versioning in dev mode * An example file IO model that generates a PowerPoint report * An example file IO model that generates an SVG output using matplotlib * Fix Git unit tests on macOS * Update model tests to cover load behavior only, repo / checkout is covered in test_repos * Update validation logic for models * Do not include period in extension for file types * Update job consistency validation to support jobs with file IO * Update graph builder to build output definitions for file outputs * Fix flow validation test * Fix error message for errors with no gRPC mapping * Update job logic to support models with file IO * Add an end-to-end test for file IO * Move common file types into a separate API module * Fix a typo in graph builder * Fix dummy test data for models (used in multiple tests) * Fix order of API imports in top level API module * Fix job validation test in -svc-orch
finos · Jan 5, 2025 · ac37c75 · ac37c75
1 parent c3ebf91
commit ac37c75
Show file tree

Hide file tree

Showing 46 changed files with 1,968 additions and 730 deletions.
diff --git a/examples/models/python/config/file_io_matplotlib.yaml b/examples/models/python/config/file_io_matplotlib.yaml
@@ -0,0 +1,9 @@
+
+job:
+  runModel:
+
+    inputs:
+      quarterly_sales: "outputs/using_data/profit_by_region.csv"
+
+    outputs:
+      sales_report: "outputs/file_io/sales_report.svg"
diff --git a/examples/models/python/config/file_io_powerpoint.yaml b/examples/models/python/config/file_io_powerpoint.yaml
@@ -0,0 +1,10 @@
+
+job:
+  runModel:
+
+    inputs:
+      quarterly_sales: "outputs/using_data/profit_by_region.csv"
+      report_template: "inputs/sales_report_template.pptx"
+
+    outputs:
+      sales_report: "outputs/file_io/sales_report.pptx"
diff --git a/examples/models/python/data/inputs/sales_report_template.pptx b/examples/models/python/data/inputs/sales_report_template.pptx
diff --git a/examples/models/python/src/tutorial/file_io_matplotlib.py b/examples/models/python/src/tutorial/file_io_matplotlib.py
@@ -0,0 +1,69 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import typing as _tp
+
+import tracdap.rt.api as trac
+
+import matplotlib
+import matplotlib.pyplot as plt
+
+import tutorial.schemas as schemas
+
+
+class MatplotlibReport(trac.TracModel):
+
+    def define_parameters(self) -> _tp.Dict[str, trac.ModelParameter]:
+
+        return trac.define_parameters()
+
+    def define_inputs(self) -> _tp.Dict[str, trac.ModelInputSchema]:
+
+        quarterly_sales_schema = trac.load_schema(schemas, "profit_by_region.csv")
+        quarterly_sales = trac.define_input(quarterly_sales_schema, label="Quarterly sales data")
+
+        return { "quarterly_sales": quarterly_sales }
+
+    def define_outputs(self) -> _tp.Dict[str, trac.ModelOutputSchema]:
+
+        sales_report = trac.define_output(trac.CommonFileTypes.SVG, label="Quarterly sales report")
+
+        return { "sales_report": sales_report }
+
+    def run_model(self, ctx: trac.TracContext):
+
+        matplotlib.use("agg")
+
+        quarterly_sales = ctx.get_pandas_table("quarterly_sales")
+
+        regions = quarterly_sales["region"]
+        values = quarterly_sales["gross_profit"]
+
+        fig = plt.figure()
+
+        plt.bar(regions, values)
+        plt.title("Profit by region report")
+        plt.xlabel("Region")
+        plt.ylabel("Gross profit")
+
+        with ctx.put_file_stream("sales_report") as sales_report:
+            plt.savefig(sales_report, format='svg')
+
+        plt.close(fig)
+
+
+if __name__ == "__main__":
+    import tracdap.rt.launch as launch
+    launch.launch_model(MatplotlibReport, "config/file_io_matplotlib.yaml", "config/sys_config.yaml")
diff --git a/examples/models/python/src/tutorial/file_io_powerpoint.py b/examples/models/python/src/tutorial/file_io_powerpoint.py
@@ -0,0 +1,77 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import typing as _tp
+
+import tracdap.rt.api as trac
+
+import pptx
+from pptx.chart.data import CategoryChartData
+from pptx.enum.chart import XL_CHART_TYPE
+from pptx.util import Inches
+
+import tutorial.schemas as schemas
+
+
+class PowerpointReport(trac.TracModel):
+
+    def define_parameters(self) -> _tp.Dict[str, trac.ModelParameter]:
+
+        return trac.define_parameters()
+
+    def define_inputs(self) -> _tp.Dict[str, trac.ModelInputSchema]:
+
+        quarterly_sales_schema = trac.load_schema(schemas, "profit_by_region.csv")
+        quarterly_sales = trac.define_input(quarterly_sales_schema, label="Quarterly sales data")
+        report_template = trac.define_input(trac.CommonFileTypes.POWERPOINT, label="Quarterly sales report template")
+
+        return {
+            "quarterly_sales": quarterly_sales,
+            "report_template": report_template }
+
+    def define_outputs(self) -> _tp.Dict[str, trac.ModelOutputSchema]:
+
+        sales_report = trac.define_output(trac.CommonFileTypes.POWERPOINT, label="Quarterly sales report")
+
+        return { "sales_report": sales_report }
+
+    def run_model(self, ctx: trac.TracContext):
+
+        quarterly_sales = ctx.get_pandas_table("quarterly_sales")
+
+        with ctx.get_file_stream("report_template") as report_template:
+            presentation = pptx.Presentation(report_template)
+
+        slide = presentation.slides.add_slide(presentation.slide_layouts[5])
+
+        title_frame = slide.shapes[0].text_frame
+        title_frame.text = 'Profit by Region Report'
+
+        # define chart data ---------------------
+        chart_data = CategoryChartData()
+        chart_data.categories = quarterly_sales["region"]
+        chart_data.add_series('Gross Profit', quarterly_sales["gross_profit"])
+
+        # add chart to slide --------------------
+        x, y, cx, cy = Inches(2), Inches(2), Inches(6), Inches(4.5)
+        slide.shapes.add_chart(XL_CHART_TYPE.COLUMN_CLUSTERED, x, y, cx, cy, chart_data)
+
+        with ctx.put_file_stream("sales_report") as sales_report:
+            presentation.save(sales_report)
+
+
+if __name__ == "__main__":
+    import tracdap.rt.launch as launch
+    launch.launch_model(PowerpointReport, "config/file_io_powerpoint.yaml", "config/sys_config.yaml")
diff --git a/tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/file.proto b/tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/file.proto
@@ -37,3 +37,10 @@ message FileDefinition {
   TagSelector storageId = 5;
   string dataItem = 6;
 }
+
+
+message FileType {
+
+  string extension = 1;
+  string mimeType = 2;
+}
diff --git a/tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/model.proto b/tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/model.proto
@@ -22,7 +22,9 @@ option java_package = "org.finos.tracdap.metadata";
 option java_multiple_files = true;
 
 import "tracdap/metadata/type.proto";
+import "tracdap/metadata/object_id.proto";
 import "tracdap/metadata/data.proto";
+import "tracdap/metadata/file.proto";
 
 
 /**
@@ -69,7 +71,12 @@ message ModelParameter {
  */
 message ModelInputSchema {
 
-    SchemaDefinition schema = 1;
+    ObjectType objectType = 6;
+
+    oneof requirement {
+        SchemaDefinition schema = 1;
+        FileType fileType = 7;
+    }
 
     optional string label = 2;
 
@@ -93,7 +100,12 @@ message ModelInputSchema {
  */
 message ModelOutputSchema {
 
-    SchemaDefinition schema = 1;
+    ObjectType objectType = 6;
+
+    oneof requirement {
+        SchemaDefinition schema = 1;
+        FileType fileType = 7;
+    }
 
     optional string label = 2;
 

diff --git a/...s/tracdap-lib-common/src/main/java/org/finos/tracdap/common/config/ServiceProperties.java b/...s/tracdap-lib-common/src/main/java/org/finos/tracdap/common/config/ServiceProperties.java
@@ -1,9 +1,10 @@
 /*
- * Copyright 2024 Accenture Global Solutions Limited
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed to the Fintech Open Source Foundation (FINOS) under one or
+ * more contributor license agreements. See the NOTICE file distributed
+ * with this work for additional information regarding copyright ownership.
+ * FINOS licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
@@ -16,8 +17,6 @@
 
 package org.finos.tracdap.common.config;
 
-import java.util.Map;
-
 public class ServiceProperties {
 
     public static final String GATEWAY_HTTP_PREFIX = "gateway.http.prefix";

diff --git a/...p-lib-common/src/main/java/org/finos/tracdap/common/grpc/DelayedExecutionInterceptor.java b/...p-lib-common/src/main/java/org/finos/tracdap/common/grpc/DelayedExecutionInterceptor.java
@@ -1,9 +1,10 @@
 /*
- * Copyright 2024 Accenture Global Solutions Limited
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed to the Fintech Open Source Foundation (FINOS) under one or
+ * more contributor license agreements. See the NOTICE file distributed
+ * with this work for additional information regarding copyright ownership.
+ * FINOS licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *

diff --git a/...libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/grpc/GrpcErrorMapping.java b/...libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/grpc/GrpcErrorMapping.java
@@ -100,7 +100,7 @@ public static StatusRuntimeException processError(Throwable error) {
 
                 log.warn("No gRPC error code mapping is available for the error {}", error.getClass().getSimpleName());
 
-                var trailers = basicErrorTrailers(Status.Code.INTERNAL, Status.INTERNAL.getDescription());
+                var trailers = basicErrorTrailers(Status.Code.INTERNAL, INTERNAL_ERROR_MESSAGE);
 
                 return Status.fromCode(Status.Code.INTERNAL)
                         .withDescription(Status.INTERNAL.getDescription())

diff --git a/tracdap-libs/tracdap-lib-test/src/main/java/org/finos/tracdap/test/meta/TestData.java b/tracdap-libs/tracdap-lib-test/src/main/java/org/finos/tracdap/test/meta/TestData.java
@@ -355,6 +355,7 @@ public static ObjectDefinition dummyModelDef() {
                 .putParameters("param1", ModelParameter.newBuilder().setParamType(TypeSystem.descriptor(BasicType.STRING)).build())
                 .putParameters("param2", ModelParameter.newBuilder().setParamType(TypeSystem.descriptor(BasicType.INTEGER)).build())
                 .putInputs("input1", ModelInputSchema.newBuilder()
+                        .setObjectType(ObjectType.DATA)
                         .setSchema(SchemaDefinition.newBuilder()
                         .setSchemaType(SchemaType.TABLE)
                         .setTable(TableSchema.newBuilder()
@@ -373,6 +374,7 @@ public static ObjectDefinition dummyModelDef() {
                                 .setFormatCode("GBP"))))
                         .build())
                 .putOutputs("output1", ModelOutputSchema.newBuilder()
+                        .setObjectType(ObjectType.DATA)
                         .setSchema(SchemaDefinition.newBuilder()
                         .setSchemaType(SchemaType.TABLE)
                         .setTable(TableSchema.newBuilder()