Common: fix PiiPseudonymizerEnrichment for arrays and improve unit te…

…sts coverage (close #334)
snowplow · Sep 15, 2020 · 2f3b421 · 2f3b421
1 parent 60af0f9
commit 2f3b421
Show file tree

Hide file tree

Showing 2 changed files with 132 additions and 7 deletions.
diff --git a/modules/common/src/test/resources/iglu-schemas/schemas/com.acme/email_sent/jsonschema/1-0-0 b/modules/common/src/test/resources/iglu-schemas/schemas/com.acme/email_sent/jsonschema/1-0-0
@@ -14,6 +14,9 @@
     },
     "emailAddress2": {
       "type": "string"
+    },
+    "emailAddress3": {
+      "type": ["string", "null"]
     }
   },
   "required": ["emailAddress", "emailAddress2"],

diff --git a/...cala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala b/...cala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala
@@ -17,21 +17,23 @@ package enrichments
 import cats.Id
 import cats.implicits._
 import cats.data.NonEmptyList
-
 import io.circe.literal._
-
 import org.joda.time.DateTime
-
 import com.snowplowanalytics.snowplow.badrows._
-import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer}
-
+import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer}
 import loaders._
 import adapters.RawEvent
+import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.pii.{
+  JsonMutators,
+  PiiJson,
+  PiiPseudonymizerEnrichment,
+  PiiStrategyPseudonymize
+}
 import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent
 import utils.Clock._
 import utils.ConversionUtils
 import enrichments.registry.{IabEnrichment, JavascriptScriptEnrichment, YauaaEnrichment}
-
+import org.apache.commons.codec.digest.DigestUtils
 import org.specs2.mutable.Specification
 import org.specs2.matcher.EitherMatchers
 
@@ -87,7 +89,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers {
               "data": {
                 "emailAddress": "hello@world.com",
                 "emailAddress2": "foo@bar.org",
-                "emailAddress3": "foo@bar.org"
+                "unallowedAdditionalField": "foo@bar.org"
               }
             }
           }"""
@@ -267,6 +269,126 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers {
       enriched.value must beRight
     }
 
+    "emit an EnrichedEvent if a PII value that needs to be hashed is an empty string" >> {
+      val parameters = Map(
+        "e" -> "ue",
+        "tv" -> "js-0.13.1",
+        "p" -> "web",
+        "co" -> """
+          {
+            "schema": "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0",
+            "data": [
+              {
+                "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
+                "data": {
+                  "emailAddress": "hello@world.com",
+                  "emailAddress2": "foo@bar.org"
+                }
+              }
+            ]
+          }
+        """,
+        "ue_pr" -> """
+          {
+            "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0",
+            "data":{
+              "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
+              "data": {
+                "emailAddress": "hello@world.com",
+                "emailAddress2": "foo@bar.org",
+                "emailAddress3": ""
+              }
+            }
+          }"""
+      )
+      val rawEvent = RawEvent(api, parameters, None, source, context)
+      val enrichmentReg = EnrichmentRegistry[Id](
+        piiPseudonymizer = PiiPseudonymizerEnrichment(
+          List(
+            PiiJson(
+              fieldMutator = JsonMutators("unstruct_event"),
+              schemaCriterion = SchemaCriterion("com.acme", "email_sent", "jsonschema", 1, 0, 0),
+              jsonPath = "$.emailAddress3"
+            )
+          ),
+          false,
+          PiiStrategyPseudonymize(
+            "MD5",
+            hashFunction = DigestUtils.sha256Hex(_: Array[Byte]),
+            "pepper123"
+          )
+        ).some
+      )
+      val enriched = EnrichmentManager.enrichEvent(
+        enrichmentReg,
+        client,
+        processor,
+        timestamp,
+        rawEvent
+      )
+      enriched.value must beRight
+    }
+
+    "emit an EnrichedEvent if a PII value that needs to be hashed is null" >> {
+      val parameters = Map(
+        "e" -> "ue",
+        "tv" -> "js-0.13.1",
+        "p" -> "web",
+        "co" -> """
+          {
+            "schema": "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0",
+            "data": [
+              {
+                "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
+                "data": {
+                  "emailAddress": "hello@world.com",
+                  "emailAddress2": "foo@bar.org"
+                }
+              }
+            ]
+          }
+        """,
+        "ue_pr" -> """
+          {
+            "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0",
+            "data":{
+              "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
+              "data": {
+                "emailAddress": "hello@world.com",
+                "emailAddress2": "foo@bar.org",
+                "emailAddress3": null
+              }
+            }
+          }"""
+      )
+      val rawEvent = RawEvent(api, parameters, None, source, context)
+      val enrichmentReg = EnrichmentRegistry[Id](
+        piiPseudonymizer = PiiPseudonymizerEnrichment(
+          List(
+            PiiJson(
+              fieldMutator = JsonMutators("unstruct_event"),
+              schemaCriterion = SchemaCriterion("com.acme", "email_sent", "jsonschema", 1, 0, 0),
+              jsonPath = "$.emailAddress3"
+            )
+          ),
+          false,
+          PiiStrategyPseudonymize(
+            "MD5",
+            hashFunction = DigestUtils.sha256Hex(_: Array[Byte]),
+            "pepper123"
+          )
+        ).some
+      )
+      val enriched = EnrichmentManager.enrichEvent(
+        enrichmentReg,
+        client,
+        processor,
+        timestamp,
+        rawEvent
+      )
+      enriched.value must beRight
+    }
+
     "have a preference of 'ua' query string parameter over user agent of HTTP header" >> {
       val qs_ua = "Mozilla/5.0 (X11; Linux x86_64; rv:75.0) Gecko/20100101 Firefox/75.0"
       val parameters = Map(