Skip to content

Commit 2ed40f1

Browse files
author
Martin Traverse
authored
Feature / Allow setting props in the metadata model (#450)
* Add props to model inputs, outputs and params * Add defining props to the static model API * Add support for auto-encoding list values * Add missing param_props in trac.P() shorthand method * Move encoding of model item props into models.py (allows encoding of items defined explicitly) * Allow adding props to TRAC objects * Validation for new props fields (not used, but still need to match the proto definition) * Include a node prop in the flow validator test * Include checking valid identifies for props in the runtime quick validation * Fix enforce platform warning
1 parent adceda6 commit 2ed40f1

File tree

14 files changed

+208
-46
lines changed

14 files changed

+208
-46
lines changed

‎gradle/base-java.gradle

+8-8
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ repositories {
2121
dependencies {
2222

2323
// Enforce a single consistent set of versions for core frameworks across the entire project
24-
implementation enforcedPlatform(group: "io.netty", name: "netty-bom", version: "${netty_version}")
25-
implementation enforcedPlatform(group: "io.grpc", name: "grpc-bom", version: "${grpc_version}")
26-
implementation enforcedPlatform(group: "com.google.protobuf", name: "protobuf-bom", version: "${proto_version}")
27-
implementation enforcedPlatform(group: "com.google.guava", name: "guava-bom", version: "${guava_version}")
28-
implementation enforcedPlatform(group: "com.google.guava", name: "guava-parent", version: "${guava_version}")
29-
implementation enforcedPlatform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "${jackson_version}")
30-
implementation enforcedPlatform(group: "org.slf4j", name: "slf4j-parent", version: "${slf4j_version}")
31-
implementation enforcedPlatform(group: "org.apache.logging.log4j", name: "log4j-bom", version: "${log4j_version}")
24+
implementation platform(group: "io.netty", name: "netty-bom", version: "${netty_version}")
25+
implementation platform(group: "io.grpc", name: "grpc-bom", version: "${grpc_version}")
26+
implementation platform(group: "com.google.protobuf", name: "protobuf-bom", version: "${proto_version}")
27+
implementation platform(group: "com.google.guava", name: "guava-bom", version: "${guava_version}")
28+
implementation platform(group: "com.google.guava", name: "guava-parent", version: "${guava_version}")
29+
implementation platform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "${jackson_version}")
30+
implementation platform(group: "org.slf4j", name: "slf4j-parent", version: "${slf4j_version}")
31+
implementation platform(group: "org.apache.logging.log4j", name: "log4j-bom", version: "${log4j_version}")
3232

3333
// Logging with SLF 4J, available in all Java modules
3434
implementation group: 'org.slf4j', name: 'slf4j-api', version: "$slf4j_version"

‎tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/model.proto

+6
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ message ModelParameter {
3434
string label = 2;
3535

3636
optional Value defaultValue = 3;
37+
38+
map<string, Value> paramProps = 4;
3739
}
3840

3941

@@ -55,6 +57,8 @@ message ModelInputSchema {
5557
optional string label = 2;
5658

5759
bool optional = 3;
60+
61+
map<string, Value> inputProps = 4;
5862
}
5963

6064
/**
@@ -76,6 +80,8 @@ message ModelOutputSchema {
7680
optional string label = 2;
7781

7882
bool optional = 3;
83+
84+
map<string, Value> outputProps = 4;
7985
}
8086

8187

‎tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/object.proto

+3
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ option java_multiple_files = true;
2222
option java_outer_classname = "ObjectProtoWrapper"; // Do not create a Java class called "Object"!
2323

2424
import "tracdap/metadata/object_id.proto";
25+
import "tracdap/metadata/type.proto";
2526
import "tracdap/metadata/data.proto";
2627
import "tracdap/metadata/model.proto";
2728
import "tracdap/metadata/flow.proto";
@@ -76,4 +77,6 @@ message ObjectDefinition {
7677
StorageDefinition storage = 8;
7778
SchemaDefinition schema = 9;
7879
}
80+
81+
map<string, Value> objectProps = 100;
7982
}

‎tracdap-libs/tracdap-lib-validation/src/main/java/org/finos/tracdap/common/validation/static_/CommonValidators.java

+8
Original file line numberDiff line numberDiff line change
@@ -717,4 +717,12 @@ public static ValidationFunction.Typed<String> uniqueContextCheck(Map<String, St
717717
};
718718
}
719719

720+
public static ValidationContext standardProps(ValidationContext ctx) {
721+
722+
return ctx
723+
.applyMapKeys(CommonValidators::identifier)
724+
.applyMapKeys(CommonValidators::notTracReserved)
725+
.applyMapValues(TypeSystemValidator::value, Value.class);
726+
}
727+
720728
}

‎tracdap-libs/tracdap-lib-validation/src/main/java/org/finos/tracdap/common/validation/static_/FlowValidator.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -188,9 +188,8 @@ public static ValidationContext flowNode(FlowNode msg, ValidationContext ctx) {
188188
.pop();
189189

190190
ctx = ctx.pushMap(FN_NODE_PROPS)
191-
.applyMapKeys(CommonValidators::identifier)
192-
.applyMapKeys(CommonValidators::notTracReserved)
193-
.applyMapValues(TypeSystemValidator::value, Value.class)
191+
.apply(CommonValidators::optional)
192+
.apply(CommonValidators::standardProps)
194193
.pop();
195194

196195
ctx = ctx.push(FN_LABEL)

‎tracdap-libs/tracdap-lib-validation/src/main/java/org/finos/tracdap/common/validation/static_/ModelValidator.java

+21-2
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,13 @@
1717
package org.finos.tracdap.common.validation.static_;
1818

1919
import org.finos.tracdap.common.validation.core.ValidationContext;
20-
import org.finos.tracdap.common.validation.core.ValidationFunction;
2120
import org.finos.tracdap.common.validation.core.ValidationType;
2221
import org.finos.tracdap.common.validation.core.Validator;
2322
import org.finos.tracdap.metadata.*;
2423

2524
import com.google.protobuf.Descriptors;
2625

2726
import java.util.HashMap;
28-
import java.util.Map;
2927

3028
import static org.finos.tracdap.common.validation.ValidationConstants.MODEL_ENTRY_POINT;
3129
import static org.finos.tracdap.common.validation.ValidationConstants.MODEL_VERSION;
@@ -49,14 +47,17 @@ public class ModelValidator {
4947
private static final Descriptors.FieldDescriptor MP_PARAM_TYPE;
5048
private static final Descriptors.FieldDescriptor MP_LABEL;
5149
private static final Descriptors.FieldDescriptor MP_DEFAULT_VALUE;
50+
private static final Descriptors.FieldDescriptor MP_PARAM_PROPS;
5251

5352
private static final Descriptors.Descriptor MODEL_INPUT_SCHEMA;
5453
private static final Descriptors.FieldDescriptor MIS_SCHEMA;
5554
private static final Descriptors.FieldDescriptor MIS_LABEL;
55+
private static final Descriptors.FieldDescriptor MIS_INPUT_PROPS;
5656

5757
private static final Descriptors.Descriptor MODEL_OUTPUT_SCHEMA;
5858
private static final Descriptors.FieldDescriptor MOS_SCHEMA;
5959
private static final Descriptors.FieldDescriptor MOS_LABEL;
60+
private static final Descriptors.FieldDescriptor MOS_OUTPUT_PROPS;
6061

6162
static {
6263

@@ -74,14 +75,17 @@ public class ModelValidator {
7475
MP_PARAM_TYPE = field(MODEL_PARAMETER, ModelParameter.PARAMTYPE_FIELD_NUMBER);
7576
MP_LABEL = field(MODEL_PARAMETER, ModelParameter.LABEL_FIELD_NUMBER);
7677
MP_DEFAULT_VALUE = field(MODEL_PARAMETER, ModelParameter.DEFAULTVALUE_FIELD_NUMBER);
78+
MP_PARAM_PROPS = field(MODEL_PARAMETER, ModelParameter.PARAMPROPS_FIELD_NUMBER);
7779

7880
MODEL_INPUT_SCHEMA = ModelInputSchema.getDescriptor();
7981
MIS_SCHEMA = field(MODEL_INPUT_SCHEMA, ModelInputSchema.SCHEMA_FIELD_NUMBER);
8082
MIS_LABEL = field(MODEL_INPUT_SCHEMA, ModelInputSchema.LABEL_FIELD_NUMBER);
83+
MIS_INPUT_PROPS = field(MODEL_INPUT_SCHEMA, ModelInputSchema.INPUTPROPS_FIELD_NUMBER);
8184

8285
MODEL_OUTPUT_SCHEMA = ModelOutputSchema.getDescriptor();
8386
MOS_SCHEMA = field(MODEL_OUTPUT_SCHEMA, ModelOutputSchema.SCHEMA_FIELD_NUMBER);
8487
MOS_LABEL = field(MODEL_OUTPUT_SCHEMA, ModelOutputSchema.LABEL_FIELD_NUMBER);
88+
MOS_OUTPUT_PROPS = field(MODEL_OUTPUT_SCHEMA, ModelOutputSchema.OUTPUTPROPS_FIELD_NUMBER);
8589
}
8690

8791
@Validator
@@ -183,6 +187,11 @@ public static ValidationContext modelParameter(ModelParameter msg, ValidationCon
183187
.apply(TypeSystemValidator::valueWithType, Value.class, msg.getParamType())
184188
.pop();
185189

190+
ctx = ctx.pushMap(MP_PARAM_PROPS)
191+
.apply(CommonValidators::optional)
192+
.apply(CommonValidators::standardProps)
193+
.pop();
194+
186195
return ctx;
187196
}
188197

@@ -199,6 +208,11 @@ public static ValidationContext modelInputSchema(ModelInputSchema msg, Validatio
199208
.apply(CommonValidators::labelLengthLimit)
200209
.pop();
201210

211+
ctx = ctx.pushMap(MIS_INPUT_PROPS)
212+
.apply(CommonValidators::optional)
213+
.apply(CommonValidators::standardProps)
214+
.pop();
215+
202216
return ctx;
203217
}
204218

@@ -215,6 +229,11 @@ public static ValidationContext modelOutputSchema(ModelOutputSchema msg, Validat
215229
.apply(CommonValidators::labelLengthLimit)
216230
.pop();
217231

232+
ctx = ctx.pushMap(MOS_OUTPUT_PROPS)
233+
.apply(CommonValidators::optional)
234+
.apply(CommonValidators::standardProps)
235+
.pop();
236+
218237
return ctx;
219238
}
220239

‎tracdap-libs/tracdap-lib-validation/src/main/java/org/finos/tracdap/common/validation/static_/ObjectValidator.java

+7
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,14 @@ public class ObjectValidator {
4545
private static final Descriptors.Descriptor OBJECT_DEFINITION;
4646
private static final Descriptors.FieldDescriptor OD_OBJECT_TYPE;
4747
private static final Descriptors.OneofDescriptor OD_DEFINITION;
48+
private static final Descriptors.FieldDescriptor OD_OBJECT_PROPS;
4849

4950
static {
5051

5152
OBJECT_DEFINITION = ObjectDefinition.getDescriptor();
5253
OD_OBJECT_TYPE = field(OBJECT_DEFINITION, ObjectDefinition.OBJECTTYPE_FIELD_NUMBER);
5354
OD_DEFINITION = field(OBJECT_DEFINITION, ObjectDefinition.DATA_FIELD_NUMBER).getContainingOneof();
55+
OD_OBJECT_PROPS = field(OBJECT_DEFINITION, ObjectDefinition.OBJECTPROPS_FIELD_NUMBER);
5456
}
5557

5658
@Validator
@@ -67,6 +69,11 @@ public static ValidationContext objectDefinition(ObjectDefinition msg, Validatio
6769
.applyRegistered()
6870
.pop();
6971

72+
ctx = ctx.pushMap(OD_OBJECT_PROPS)
73+
.apply(CommonValidators::optional)
74+
.apply(CommonValidators::standardProps)
75+
.pop();
76+
7077
return ctx;
7178
}
7279

‎tracdap-libs/tracdap-lib-validation/src/test/java/org/finos/tracdap/common/validation/static_/FlowValidatorTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ void basicFlow_ok1() {
3939
.putNodes("model_1", FlowNode.newBuilder().setNodeType(FlowNodeType.MODEL_NODE)
4040
.addInputs("input_1").addInputs("input_2")
4141
.addOutputs("output_1")
42+
.putNodeProps("sample_prop", MetadataCodec.encodeValue(2.0))
4243
.build())
4344
.putNodes("output_1", FlowNode.newBuilder().setNodeType(FlowNodeType.OUTPUT_NODE).build())
4445

‎tracdap-runtime/python/src/tracdap/rt/_impl/models.py

+25
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,15 @@ def scan_model(self, model_stub: _meta.ModelDefinition, model_class: _api.TracMo
215215

216216
for name, param in model_def.parameters.items():
217217
self.__log.info(f"Parameter [{name}] - {param.paramType.basicType.name}")
218+
param.paramProps = self._encoded_props(param.paramProps, "parameter", name)
218219

219220
for name, schema in model_def.inputs.items():
220221
self.__log.info(f"Input [{name}] - {schema.schema.schemaType.name}")
222+
schema.inputProps = self._encoded_props(schema.inputProps, "input", name)
221223

222224
for name, schema in model_def.outputs.items():
223225
self.__log.info(f"Output [{name}] - {schema.schema.schemaType.name}")
226+
schema.outputProps = self._encoded_props(schema.outputProps, "input", name)
224227

225228
return model_def
226229

@@ -231,3 +234,25 @@ def scan_model(self, model_stub: _meta.ModelDefinition, model_class: _api.TracMo
231234

232235
self.__log.error(msg, exc_info=True)
233236
raise _ex.EModelValidation(msg) from e
237+
238+
@staticmethod
239+
def _encoded_props(
240+
raw_props: tp.Dict[str, tp.Any],
241+
item_type: str, item_name: str) \
242+
-> tp.Dict[str, _meta.Value]:
243+
244+
if raw_props is None:
245+
return dict()
246+
247+
encoded_props = dict()
248+
249+
for key, raw_value in raw_props.items():
250+
251+
if raw_value is None:
252+
raise _ex.EModelValidation(f"Invalid null property [{key}] for {item_type} [{item_name}]")
253+
elif isinstance(raw_value, _meta.Value):
254+
encoded_props[key] = raw_value
255+
else:
256+
encoded_props[key] = _types.MetadataCodec.encode_value(raw_value)
257+
258+
return encoded_props

‎tracdap-runtime/python/src/tracdap/rt/_impl/static_api.py

+23-9
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,14 @@ def define_attributes(
7171

7272
def define_parameter(
7373
self, param_name: str, param_type: _tp.Union[_meta.TypeDescriptor, _meta.BasicType],
74-
label: str, default_value: _tp.Optional[_tp.Any] = None) \
74+
label: str, default_value: _tp.Optional[_tp.Any] = None,
75+
*, param_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
7576
-> _Named[_meta.ModelParameter]:
7677

77-
_val.validate_signature(self.define_parameter, param_name, param_type, label, default_value)
78+
_val.validate_signature(
79+
self.define_parameter,
80+
param_name, param_type, label, default_value,
81+
param_props=param_props)
7882

7983
if isinstance(param_type, _meta.TypeDescriptor):
8084
param_type_descriptor = param_type
@@ -88,7 +92,9 @@ def define_parameter(
8892
msg = f"Default value for parameter [{param_name}] does not match the declared type"
8993
raise _ex.EModelValidation(msg) from e
9094

91-
return _Named(param_name, _meta.ModelParameter(param_type_descriptor, label, default_value))
95+
return _Named(param_name, _meta.ModelParameter(
96+
param_type_descriptor, label, default_value,
97+
paramProps=param_props))
9298

9399
def define_parameters(
94100
self, *params: _tp.Union[_Named[_meta.ModelParameter], _tp.List[_Named[_meta.ModelParameter]]]) \
@@ -149,24 +155,32 @@ def load_schema(
149155
def define_input_table(
150156
self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
151157
label: _tp.Optional[str] = None,
152-
optional: bool = False) \
158+
optional: bool = False,
159+
input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
153160
-> _meta.ModelInputSchema:
154161

155-
_val.validate_signature(self.define_input_table, *fields, label=label, optional=optional)
162+
_val.validate_signature(
163+
self.define_input_table, *fields,
164+
label=label, optional=optional,
165+
input_props=input_props)
156166

157167
schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
158-
return _meta.ModelInputSchema(schema=schema_def, label=label, optional=optional)
168+
return _meta.ModelInputSchema(schema=schema_def, label=label, optional=optional, inputProps=input_props)
159169

160170
def define_output_table(
161171
self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
162172
label: _tp.Optional[str] = None,
163-
optional: bool = False) \
173+
optional: bool = False,
174+
output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
164175
-> _meta.ModelOutputSchema:
165176

166-
_val.validate_signature(self.define_output_table, *fields, label=label, optional=optional)
177+
_val.validate_signature(
178+
self.define_output_table, *fields,
179+
label=label, optional=optional,
180+
output_props=output_props)
167181

168182
schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
169-
return _meta.ModelOutputSchema(schema=schema_def, label=label, optional=optional)
183+
return _meta.ModelOutputSchema(schema=schema_def, label=label, optional=optional, outputProps=output_props)
170184

171185
@staticmethod
172186
def _build_named_dict(

‎tracdap-runtime/python/src/tracdap/rt/_impl/type_system.py

+17
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,23 @@ def encode_value(cls, value: tp.Any) -> _meta.Value:
171171
type_desc = _meta.TypeDescriptor(_meta.BasicType.DATE)
172172
return _meta.Value(type_desc, dateValue=_meta.DateValue(value.isoformat()))
173173

174+
if isinstance(value, list):
175+
176+
if len(value) == 0:
177+
raise _ex.ETracInternal("Cannot encode an empty list")
178+
179+
array_raw_type = type(value[0])
180+
array_trac_type = TypeMapping.python_to_trac(array_raw_type)
181+
182+
if any(map(lambda x: type(x) != array_raw_type, value)):
183+
raise _ex.ETracInternal("Cannot encode a list with values of different types")
184+
185+
encoded_items = list(map(lambda x: cls.convert_value(x, array_trac_type), value))
186+
187+
return _meta.Value(
188+
_meta.TypeDescriptor(_meta.BasicType.ARRAY, arrayType=array_trac_type),
189+
arrayValue=_meta.ArrayValue(encoded_items))
190+
174191
raise _ex.ETracInternal(f"Value type [{type(value)}] is not supported yet")
175192

176193
@classmethod

‎tracdap-runtime/python/src/tracdap/rt/_impl/validation.py

+10
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,9 @@ def _check_parameters(cls, parameters):
320320
else:
321321
cls._check_label(param.label, param_name)
322322

323+
if param.paramProps is not None:
324+
cls._valid_identifiers(param.paramProps.keys(), "entry in param props")
325+
323326
@classmethod
324327
def _check_inputs_or_outputs(cls, inputs_or_outputs):
325328

@@ -340,6 +343,13 @@ def _check_inputs_or_outputs(cls, inputs_or_outputs):
340343
label = input_schema.label
341344
cls._check_label(label, input_name)
342345

346+
if isinstance(input_schema, meta.ModelInputSchema):
347+
if input_schema.inputProps is not None:
348+
cls._valid_identifiers(input_schema.inputProps.keys(), "entry in input props")
349+
else:
350+
if input_schema.outputProps is not None:
351+
cls._valid_identifiers(input_schema.outputProps.keys(), "entry in output props")
352+
343353
@classmethod
344354
def _check_single_field(cls, field: meta.FieldSchema, property_type):
345355

‎tracdap-runtime/python/src/tracdap/rt/api/hook.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,8 @@ def define_attribute(
8080
@_abc.abstractmethod
8181
def define_parameter(
8282
self, param_name: str, param_type: _tp.Union[_meta.TypeDescriptor, _meta.BasicType],
83-
label: str, default_value: _tp.Optional[_tp.Any] = None) \
83+
label: str, default_value: _tp.Optional[_tp.Any] = None,
84+
*, param_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
8485
-> _Named[_meta.ModelParameter]:
8586

8687
pass
@@ -121,7 +122,8 @@ def load_schema(
121122
def define_input_table(
122123
self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
123124
label: _tp.Optional[str] = None,
124-
optional: bool = False) \
125+
optional: bool = False,
126+
input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
125127
-> _meta.ModelInputSchema:
126128

127129
pass
@@ -130,7 +132,8 @@ def define_input_table(
130132
def define_output_table(
131133
self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
132134
label: _tp.Optional[str] = None,
133-
optional: bool = False) \
135+
optional: bool = False,
136+
output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
134137
-> _meta.ModelOutputSchema:
135138

136139
pass

0 commit comments

Comments
 (0)