PR for issue #291 - 0.6 API Updates - Mandatory notNull flag in field schemas

examples/apps/javascript/src/hello_world.js

@@ -33,7 +33,7 @@ export function createSchema() {
         table: {
             fields: [
                 {
-                    fieldName: "customer_id", fieldType: tracdap.STRING, businessKey: true,
+                    fieldName: "customer_id", fieldType: tracdap.STRING, businessKey: true, notNull: true,
                     label: "Unique customer account number"
                 },
                 {

tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/data.proto

@@ -22,87 +22,9 @@ option java_multiple_files = true;
 
 import "tracdap/metadata/type.proto";
 import "tracdap/metadata/object_id.proto";
+import "tracdap/metadata/schema_proto.proto";
 
 
-/**
- * Enumeration of the available types of data schema
- *
- * Currently only table schemas are supported, other schema types may be added later.
- *
- * @see SchemaDefinition
- */
-enum SchemaType {
-
-    SCHEMA_TYPE_NOT_SET = 0;
-    TABLE = 1;
-}
-
-message FieldSchema {
-
-    string fieldName = 1;
-    sint32 fieldOrder = 2;
-    BasicType fieldType = 3;
-
-    string label = 4;
-
-    bool businessKey = 5;
-    bool categorical = 6;
-
-    // This could become mandatory with the next metadata update
-    optional bool notNull = 8;
-
-    optional string formatCode = 7;
-}
-
-
-message TableSchema {
-
-    repeated FieldSchema fields = 1;
-}
-
-
-/**
- * A schema definition describes the schema of a dataset
- *
- * Schema definitions can be top level objects (a type of object definition),
- * in which case they can be referred to by multiple data definitions. Alternatively
- * they can be embedded in a data definition to create datasets with one-off schemas.
- *
- * A table schema describes the schema of a tabular data set. Other schema types may
- * be added later, e.g. for matrices, tensors, curves, surfaces and structured datasets.
- *
- * @see DataDefinition
- */
-message SchemaDefinition {
-
-    SchemaType schemaType = 1;
-    PartType partType = 2;
-
-    oneof schemaTypeDefinition {
-
-        TableSchema table = 3;
-    }
-}
-
-
-
-
-enum PartType {
-    PART_ROOT = 0;
-    PART_BY_RANGE = 1;
-    PART_BY_VALUE = 2;
-}
-
-
-message PartKey {
-
-    string opaqueKey = 1;
-
-    PartType partType = 2;
-    repeated Value partValues = 3;
-    optional Value partRangeMin = 4;
-    optional Value partRangeMax = 5;
-}
 
 
 message DataDefinition {

tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/model.proto

@@ -22,6 +22,7 @@ option java_multiple_files = true;
 
 import "tracdap/metadata/type.proto";
 import "tracdap/metadata/data.proto";
+import "tracdap/metadata/schema_proto.proto";
 
 
 /**

tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/object.proto

@@ -29,6 +29,7 @@ import "tracdap/metadata/job.proto";
 import "tracdap/metadata/file.proto";
 import "tracdap/metadata/custom.proto";
 import "tracdap/metadata/stoarge.proto";
+import "tracdap/metadata/schema_proto.proto";
 
 
 /**

tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/schema_proto.proto

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020 Accenture Global Solutions Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = 'proto3';
+package tracdap.metadata;
+
+option java_package = "org.finos.tracdap.metadata";
+option java_multiple_files = true;
+
+import "tracdap/metadata/type.proto";
+
+/**
+ * Enumeration of the available types of data schema
+ *
+ * Currently only table schemas are supported, other schema types may be added later.
+ *
+ * @see SchemaDefinition
+ */
+enum SchemaType {
+
+  SCHEMA_TYPE_NOT_SET = 0;
+  TABLE = 1;
+}
+
+message FieldSchema {
+
+  string fieldName = 1;
+  sint32 fieldOrder = 2;
+  BasicType fieldType = 3;
+
+  string label = 4;
+
+  bool businessKey = 5;
+  bool categorical = 6;
+
+  bool notNull = 8;
+
+  optional string formatCode = 7;
+}
+
+
+message TableSchema {
+
+  repeated FieldSchema fields = 1;
+}
+
+
+/**
+ * A schema definition describes the schema of a dataset
+ *
+ * Schema definitions can be top level objects (a type of object definition),
+ * in which case they can be referred to by multiple data definitions. Alternatively
+ * they can be embedded in a data definition to create datasets with one-off schemas.
+ *
+ * A table schema describes the schema of a tabular data set. Other schema types may
+ * be added later, e.g. for matrices, tensors, curves, surfaces and structured datasets.
+ *
+ * @see DataDefinition
+ */
+message SchemaDefinition {
+
+  SchemaType schemaType = 1;
+  PartType partType = 2;
+
+  oneof schemaTypeDefinition {
+
+    TableSchema table = 3;
+  }
+}
+
+
+enum PartType {
+  PART_ROOT = 0;
+  PART_BY_RANGE = 1;
+  PART_BY_VALUE = 2;
+}
+
+
+message PartKey {
+
+  string opaqueKey = 1;
+
+  PartType partType = 2;
+  repeated Value partValues = 3;
+  optional Value partRangeMin = 4;
+  optional Value partRangeMax = 5;
+}

tracdap-api/tracdap-services/src/main/proto/tracdap/api/data.proto

@@ -25,6 +25,7 @@ import "tracdap/metadata/object_id.proto";
 import "tracdap/metadata/data.proto";
 import "tracdap/metadata/file.proto";
 import "tracdap/metadata/tag_update.proto";
+import "tracdap/metadata/schema_proto.proto";
 
 // Force dependency ordering - API protos depend on all of metadata
 import "tracdap/metadata/tag.proto";

tracdap-libs/tracdap-lib-test/src/main/java/org/finos/tracdap/test/meta/TestData.java

@@ -143,22 +143,26 @@ public static ObjectDefinition dummyDataDef() {
                         .setFieldName("transaction_id")
                         .setFieldType(BasicType.STRING)
                         .setFieldOrder(0)
-                        .setBusinessKey(true))
+                        .setBusinessKey(true)
+                        .setNotNull(true))
                 .addFields(FieldSchema.newBuilder()
                         .setFieldName("customer_id")
                         .setFieldType(BasicType.STRING)
                         .setFieldOrder(1)
-                        .setBusinessKey(true))
+                        .setBusinessKey(true)
+                        .setNotNull(true))
                 .addFields(FieldSchema.newBuilder()
                         .setFieldName("order_date")
                         .setFieldType(BasicType.DATE)
                         .setFieldOrder(2)
-                        .setBusinessKey(true))
+                        .setBusinessKey(true)
+                        .setNotNull(true))
                 .addFields(FieldSchema.newBuilder()
                         .setFieldName("widgets_ordered")
                         .setFieldType(BasicType.INTEGER)
                         .setFieldOrder(3)
-                        .setBusinessKey(true))))
+                        .setBusinessKey(true)
+                        .setNotNull(true))))
 
                 .putParts("part-root", DataDefinition.Part.newBuilder()
                         .setPartKey(PartKey.newBuilder()
@@ -283,7 +287,8 @@ public static ObjectDefinition dummyModelDef() {
                         .addFields(FieldSchema.newBuilder()
                                 .setFieldName("field1")
                                 .setFieldType(BasicType.DATE)
-                                .setBusinessKey(true))
+                                .setBusinessKey(true)
+                                .setNotNull(true))
                         .addFields(FieldSchema.newBuilder()
                                 .setFieldName("field2")
                                 .setFieldType(BasicType.STRING)

tracdap-libs/tracdap-lib-validation/src/main/java/org/finos/tracdap/common/validation/static_/SchemaValidator.java

@@ -199,7 +199,7 @@ public static ValidationContext fieldSchema(FieldSchema field, ValidationContext
 
         // If the notNull field is set on a business key, it must be set to true
 
-        if (field.getBusinessKey() && field.hasNotNull() && !field.getNotNull()) {
+        if (field.getBusinessKey() && !field.getNotNull()) {
 
             var err = String.format(
                     "Schema field [%s] cannot have notNull == false because it is a business key",

tracdap-runtime/python/test/tracdap_test/rt/jobs/test_core_jobs.py

@@ -185,7 +185,7 @@ def _build_run_model_job_config(self):
                     "customer_loans": meta.ModelInputSchema(meta.SchemaDefinition(
                         schemaType=meta.SchemaType.TABLE,
                         table=meta.TableSchema(fields=[
-                            meta.FieldSchema("id", fieldType=meta.BasicType.STRING, businessKey=True),
+                            meta.FieldSchema("id", fieldType=meta.BasicType.STRING, businessKey=True, notNull=True),
                             meta.FieldSchema("loan_amount", fieldType=meta.BasicType.DECIMAL),
                             meta.FieldSchema("total_pymnt", fieldType=meta.BasicType.DECIMAL),
                             meta.FieldSchema("region", fieldType=meta.BasicType.STRING, categorical=True),

tracdap-services/tracdap-svc-orch/src/test/java/org/finos/tracdap/svc/orch/jobs/RunFlowTest.java

@@ -159,7 +159,8 @@ void createSchemas() {
                         .addFields(FieldSchema.newBuilder()
                                 .setFieldName("id")
                                 .setFieldType(BasicType.STRING)
-                                .setBusinessKey(true))
+                                .setBusinessKey(true)
+                                .setNotNull(true))
                         .addFields(FieldSchema.newBuilder()
                                 .setFieldName("loan_amount")
                                 .setFieldType(BasicType.DECIMAL))

tracdap-services/tracdap-svc-orch/src/test/java/org/finos/tracdap/svc/orch/jobs/RunModelTest.java

@@ -83,7 +83,8 @@ void loadInputData() throws Exception {
                 .addFields(FieldSchema.newBuilder()
                         .setFieldName("id")
                         .setFieldType(BasicType.STRING)
-                        .setBusinessKey(true))
+                        .setBusinessKey(true)
+                        .setNotNull(true))
                 .addFields(FieldSchema.newBuilder()
                         .setFieldName("loan_amount")
                         .setFieldType(BasicType.DECIMAL))