feat: support of ChangeType in MCE and MAE (#131)

* support of ChangeType in MCE and MAE * make changetype optional field * add proper doc for delete change type
linkedin · Oct 22, 2021 · 2642177 · 2642177
1 parent 7f8409e
commit 2642177
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 0 deletions.
diff --git a/core-models/src/main/pegasus/com/linkedin/common/ChangeType.pdl b/core-models/src/main/pegasus/com/linkedin/common/ChangeType.pdl
@@ -0,0 +1,16 @@
+namespace com.linkedin.common
+
+/**
+ * Descriptor for a change action. For every type, a new version of metadata will be created
+ */
+enum ChangeType {
+  /**
+   * create new metadata if it doesn't exist, update if it already exists
+   */
+  UPSERT
+
+  /**
+   * delete the latest version of metadata, if it exists
+   */
+  DELETE
+}
diff --git a/gradle-plugins/metadata-events-generator-lib/src/main/resources/MetadataAuditEvent.rythm b/gradle-plugins/metadata-events-generator-lib/src/main/resources/MetadataAuditEvent.rythm
@@ -5,6 +5,7 @@
 namespace com.linkedin.mxe@(nameSpace)
 
 import com.linkedin.avro2pegasus.events.KafkaAuditHeader
+import com.linkedin.common.ChangeType
 import @entityUrn
 import @eventSpec.getFullValueType()
 
@@ -33,4 +34,9 @@ record MetadataAuditEvent {
    * Aspect of the @eventSpec.getValueType() after the update.
    */
   newValue: @eventSpec.getValueType()
+
+  /**
+   * Change type.
+   */
+  changeType: optional ChangeType
 }
diff --git a/gradle-plugins/metadata-events-generator-lib/src/main/resources/MetadataChangeEvent.rythm b/gradle-plugins/metadata-events-generator-lib/src/main/resources/MetadataChangeEvent.rythm
@@ -5,6 +5,7 @@
 namespace com.linkedin.mxe@(nameSpace)
 
 import com.linkedin.avro2pegasus.events.KafkaAuditHeader
+import com.linkedin.common.ChangeType
 import @entityUrn
 import @eventSpec.getFullValueType()
 
@@ -28,4 +29,9 @@ record MetadataChangeEvent {
    * Value of the proposed @eventSpec.getValueType() change.
    */
   proposedValue: optional @eventSpec.getValueType()
+
+  /**
+   * Change type.
+   */
+  changeType: optional ChangeType
 }
diff --git a/...es/expectedSchemas/pegasus/com/linkedin/mxe/bar/annotatedAspectBar/MetadataAuditEvent.pdl b/...es/expectedSchemas/pegasus/com/linkedin/mxe/bar/annotatedAspectBar/MetadataAuditEvent.pdl
@@ -1,6 +1,7 @@
 namespace com.linkedin.mxe.bar.annotatedAspectBar
 
 import com.linkedin.avro2pegasus.events.KafkaAuditHeader
+import com.linkedin.common.ChangeType
 import com.linkedin.testing.BarUrn
 import com.linkedin.testing.AnnotatedAspectBar
 
@@ -29,4 +30,9 @@ record MetadataAuditEvent {
    * Aspect of the AnnotatedAspectBar after the update.
    */
   newValue: AnnotatedAspectBar
+
+  /**
+   * Change type.
+   */
+  changeType: optional ChangeType
 }
diff --git a/...s/expectedSchemas/pegasus/com/linkedin/mxe/bar/annotatedAspectBar/MetadataChangeEvent.pdl b/...s/expectedSchemas/pegasus/com/linkedin/mxe/bar/annotatedAspectBar/MetadataChangeEvent.pdl
@@ -1,6 +1,7 @@
 namespace com.linkedin.mxe.bar.annotatedAspectBar
 
 import com.linkedin.avro2pegasus.events.KafkaAuditHeader
+import com.linkedin.common.ChangeType
 import com.linkedin.testing.BarUrn
 import com.linkedin.testing.AnnotatedAspectBar
 
@@ -24,4 +25,9 @@ record MetadataChangeEvent {
    * Value of the proposed AnnotatedAspectBar change.
    */
   proposedValue: optional AnnotatedAspectBar
+
+  /**
+   * Change type.
+   */
+  changeType: optional ChangeType
 }
diff --git a/...rc/test/groovy/com/linkedin/metadata/gradle/MetadataEventsGeneratorPluginIntegTest.groovy b/...rc/test/groovy/com/linkedin/metadata/gradle/MetadataEventsGeneratorPluginIntegTest.groovy
@@ -94,6 +94,7 @@ class MetadataEventsGeneratorPluginIntegTest extends Specification {
       namespace com.linkedin.mxe.foo.testAspect
 
       import com.linkedin.avro2pegasus.events.KafkaAuditHeader
+      import com.linkedin.common.ChangeType
       import com.linkedin.testing.FooUrn
       import com.linkedin.metadata.test.aspects.TestAspect
 
@@ -117,12 +118,18 @@ class MetadataEventsGeneratorPluginIntegTest extends Specification {
          * Value of the proposed TestAspect change.
          */
         proposedValue: optional TestAspect
+
+        /**
+         * Change type.
+         */
+        changeType: optional ChangeType
       }'''.stripIndent()
 
     projectFile('build/my-dummy-module/generateMetadataEventsSchema/com/linkedin/mxe/foo/testAspect/MetadataAuditEvent.pdl').text == '''\
       namespace com.linkedin.mxe.foo.testAspect
 
       import com.linkedin.avro2pegasus.events.KafkaAuditHeader
+      import com.linkedin.common.ChangeType
       import com.linkedin.testing.FooUrn
       import com.linkedin.metadata.test.aspects.TestAspect
 
@@ -151,6 +158,11 @@ class MetadataEventsGeneratorPluginIntegTest extends Specification {
          * Aspect of the TestAspect after the update.
          */
         newValue: TestAspect
+
+        /**
+         * Change type.
+         */
+        changeType: optional ChangeType
       }'''.stripIndent()
 
     projectFile('build/my-dummy-module/generateMetadataEventsSchema/com/linkedin/mxe/foo/testAspect/FailedMetadataChangeEvent.pdl').text == '''\