examples with custom privacy model

metadata-ingestion/src/datahub/cli/cli_utils.py

@@ -300,7 +300,7 @@ def post_rollback_endpoint(
     unsafe_entity_count = summary.get("unsafeEntitiesCount", 0)
     unsafe_entities = summary.get("unsafeEntities", [])
     rolled_back_aspects = list(
-        filter(lambda row: row["runId"] == payload_obj["runId"], rows)
+        filter(lambda row: row.get("runId") == payload_obj.get("runId"), rows)
     )
 
     if len(rows) == 0:
@@ -424,7 +424,8 @@ def get_urns_by_filter(
         "filter": {"or": [{"and": filter_criteria}]},
     }
     payload = json.dumps(search_body)
-    log.debug(payload)
+    log.debug(f"url={url}")
+    log.debug(f"payload={payload}")
     response: Response = session.post(url, payload)
     if response.status_code == 200:
         assert response._content

metadata-models-custom/example_loader/build.sh

@@ -0,0 +1,2 @@
+#!/bin/zsh
+../gradlew -PprojVersion=0.0.2 clean build install

metadata-models-custom/example_loader/loader.sh

@@ -0,0 +1,2 @@
+#!/bin/zsh
+datahub put --urn "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" --aspect privacyInfo -d privacy_info.json

metadata-models-custom/example_loader/privacy_info.json

@@ -0,0 +1,5 @@
+{
+    "policy_id": "gdpr_policy",
+    "privacy_compliant": "False",
+    "privacy_last_enforcement_time": 1000000
+}

metadata-models-custom/registry/entity-registry.yaml

@@ -3,3 +3,4 @@ entities:
   - name: dataset
     aspects:
       - customDataQualityRules
+      - privacyInfo

metadata-models-custom/src/main/pegasus/com/mycompany/privacy/PrivacyInfo.pdl

@@ -0,0 +1,21 @@
+namespace com.mycompany.privacy
+
+@Aspect = {
+    "name": "privacyInfo",
+    "autoRender": true,
+    "renderSpec": {
+    "displayType": "properties", // or properties
+    "displayName": "Privacy"
+  }
+}
+record PrivacyInfo {
+  policy_id: optional string
+
+  @Searchable = {
+    "fieldType": "KEYWORD",
+    "addToFilters": true,
+  }
+  privacy_compliant: optional string
+
+  privacy_last_enforcement_time: optional long
+}