Common: bump YAUAA to 5.19 (close #314)

modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala

@@ -202,7 +202,11 @@ object EnrichmentConf {
       WeatherEnrichment[F](this)
   }
 
-  final case class YauaaConf(schemaKey: SchemaKey, cacheSize: Option[Int]) extends EnrichmentConf {
-    def enrichment: YauaaEnrichment = YauaaEnrichment(cacheSize)
+  final case class YauaaConf(
+    schemaKey: SchemaKey,
+    model: Int,
+    cacheSize: Option[Int]
+  ) extends EnrichmentConf {
+    def enrichment: YauaaEnrichment = YauaaEnrichment(model, cacheSize)
   }
 }

modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala

@@ -29,7 +29,7 @@ import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils
 
 /** Companion object to create an instance of YauaaEnrichment from the configuration. */
 object YauaaEnrichment extends ParseableEnrichment {
-  val supportedSchema =
+  val supportedSchema: SchemaCriterion =
     SchemaCriterion(
       "com.snowplowanalytics.snowplow.enrichments",
       "yauaa_enrichment_config",
@@ -38,6 +38,25 @@ object YauaaEnrichment extends ParseableEnrichment {
       0
     )
 
+  /**
+   * Enrichment can produce `yauaa_context` with following models
+   * 1-0-0 comes from original YAUAA 5.8
+   * 2-0-0 comes from YAUAA 5.19 and adds two new fields:
+   * `operatingSystemNameVersionMajor` and `operatingSystemVersionMajor`
+   * For 1-0-0 those fields are removed during context construction
+   */
+  val SupportedContextModels = List(1, 2)
+
+  /** Default `yauaa_context` MODEL from [[SupportedContextModels]] */
+  val DefaultContextModel = 1
+
+  /** Fields that were added in YAUUA 5.19 and not supported by `iglu:nl.basjes/yauaa_context/1-0-0` */
+  val Model2Fields: List[String] =
+    List(UserAgent.OPERATING_SYSTEM_NAME_VERSION_MAJOR, UserAgent.OPERATING_SYSTEM_VERSION_MAJOR)
+
+  val DefaultDeviceClass = "Unknown"
+  val DefaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> DefaultDeviceClass)
+
   /**
    * Creates a YauaaConf instance from a JValue containing the configuration of the enrichment.
    *
@@ -54,7 +73,9 @@ object YauaaEnrichment extends ParseableEnrichment {
     (for {
       _ <- isParseable(c, schemaKey)
       cacheSize <- CirceUtils.extract[Option[Int]](c, "parameters", "cacheSize").toEither
-    } yield YauaaConf(schemaKey, cacheSize)).toValidatedNel
+      model <- CirceUtils.extract[Option[Int]](c, "parameters", "model").toEither.map(_.getOrElse(DefaultContextModel))
+      _ <- checkModel(model)
+    } yield YauaaConf(schemaKey, model, cacheSize)).toValidatedNel
 
   /** Helper to decapitalize a string. Used for the names of the fields returned in the context. */
   def decapitalize(s: String): String =
@@ -63,14 +84,19 @@ object YauaaEnrichment extends ParseableEnrichment {
       case _ if s.length == 1 => s.toLowerCase
       case _ => s.charAt(0).toLower + s.substring(1)
     }
+
+  private def checkModel(model: Int): Either[String, Unit] =
+    if (SupportedContextModels.contains(model)) ().asRight
+    else s"""YAUAA enrichment context with model $model is not supported. Choose from: ${SupportedContextModels.mkString(", ")}""".asLeft
 }
 
 /**
  * Class for YAUAA enrichment, which tries to parse and analyze the user agent string
  * and extract as many relevant attributes as possible, like for example the device class.
+ * @param model version of yauaa_context produced, `1` and `2` are supported currently
  * @param cacheSize Amount of user agents already parsed that stay in cache for faster parsing.
  */
-final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment {
+final case class YauaaEnrichment(model: Int, cacheSize: Option[Int]) extends Enrichment {
   import YauaaEnrichment.decapitalize
 
   private val uaa: UserAgentAnalyzer = {
@@ -81,10 +107,7 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment {
     a
   }
 
-  val outputSchema = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 0))
-
-  val defaultDeviceClass = "Unknown"
-  val defaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> defaultDeviceClass)
+  val outputSchema: SchemaKey = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(model, 0, 0))
 
   /**
    * Gets the result of YAUAA user agent analysis as self-describing JSON, for a specific event.
@@ -102,11 +125,16 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment {
   def parseUserAgent(userAgent: String): Map[String, String] =
     userAgent match {
       case null | "" =>
-        defaultResult
+        YauaaEnrichment.DefaultResult
       case _ =>
         val parsedUA = uaa.parse(userAgent)
-        parsedUA.getAvailableFieldNames.asScala
+        parsedUA.getAvailableFieldNamesSorted.asScala
+          .filter(stripNewFields)
           .map(field => decapitalize(field) -> parsedUA.getValue(field))
           .toMap
     }
+
+  /** Predicate to remove YAUAA 5.19 fields if `iglu:nl.basjes/yauaa_context/1-0-0` is used */
+  def stripNewFields(field: String): Boolean =
+    !(model == 1 && YauaaEnrichment.Model2Fields.contains(field))
 }

modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala

@@ -37,7 +37,7 @@ import enrichments.registry.JavascriptScriptEnrichment
 import enrichments.registry.YauaaEnrichment
 
 class EnrichmentManagerSpec extends Specification with EitherMatchers {
-  val enrichmentReg = EnrichmentRegistry[Id](yauaa = Some(YauaaEnrichment(None)))
+  val enrichmentReg = EnrichmentRegistry[Id](yauaa = Some(YauaaEnrichment(YauaaEnrichment.DefaultContextModel, None)))
   val client = SpecHelpers.client
   val processor = Processor("ssc-tests", "0.0.0")
   val timestamp = DateTime.now()

modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala

@@ -12,11 +12,11 @@
  */
 package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry
 
-import io.circe.parser._
+import cats.data.NonEmptyList
+
 import io.circe.literal._
 
 import nl.basjes.parse.useragent.UserAgent
-
 import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData}
 
 import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.YauaaConf
@@ -28,7 +28,11 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers {
 
   import YauaaEnrichment.decapitalize
 
-  val yauaaEnrichment = YauaaEnrichment(None)
+  /** Default enrichment with 1-0-0 context */
+  val yauaaEnrichment = YauaaEnrichment(YauaaEnrichment.DefaultContextModel, None)
+
+  /** Enrichment with 2-0-0 context coming from YAUAA 5.19 */
+  val yauaaEnrichment2 = YauaaEnrichment(2, None)
 
   // Devices
   val uaIpad =
@@ -68,11 +72,11 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers {
 
   "YAUAA enrichment should" >> {
     "return default value for null" >> {
-      yauaaEnrichment.parseUserAgent(null) shouldEqual yauaaEnrichment.defaultResult
+      yauaaEnrichment.parseUserAgent(null) shouldEqual YauaaEnrichment.DefaultResult
     }
 
     "return default value for empty user agent" >> {
-      yauaaEnrichment.parseUserAgent("") shouldEqual yauaaEnrichment.defaultResult
+      yauaaEnrichment.parseUserAgent("") shouldEqual YauaaEnrichment.DefaultResult
     }
 
     "detect correctly DeviceClass" >> {
@@ -185,7 +189,7 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers {
       )
     }
 
-    "create a JSON with the schema and the data" >> {
+    "create a JSON with the schema 1-0-0 and the data" >> {
       val expected =
         SelfDescribingData(
           yauaaEnrichment.outputSchema,
@@ -201,6 +205,47 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers {
         )
       yauaaEnrichment.getYauaaContext("") shouldEqual defaultJson
     }
+
+    "create a JSON with the schema 2-0-0 and the data" >> {
+      val expected =
+        SelfDescribingData(
+          SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(2, 0, 0)),
+          json"""{
+            "deviceBrand":"Samsung",
+            "deviceName":"Samsung SM-G960F",
+            "layoutEngineNameVersion":"Blink 62.0",
+            "operatingSystemNameVersion":"Android 8.0.0",
+            "operatingSystemVersionBuild":"R16NW",
+            "layoutEngineNameVersionMajor":"Blink 62",
+            "operatingSystemName":"Android",
+            "agentVersionMajor":"62",
+            "layoutEngineVersionMajor":"62",
+            "deviceClass":"Phone",
+            "agentNameVersionMajor":"Chrome 62",
+            "operatingSystemClass":"Mobile",
+            "layoutEngineName":"Blink",
+            "agentName":"Chrome",
+            "agentVersion":"62.0.3202.84",
+            "layoutEngineClass":"Browser",
+            "agentNameVersion":"Chrome 62.0.3202.84",
+            "operatingSystemVersion":"8.0.0",
+            "agentClass":"Browser",
+            "layoutEngineVersion":"62.0",
+
+            "operatingSystemVersionMajor" : "8",
+            "operatingSystemNameVersionMajor" : "Android 8"
+          }"""
+        )
+      val actual = yauaaEnrichment2.getYauaaContext(uaGalaxyS9)
+      actual shouldEqual expected
+
+      val defaultJson =
+        SelfDescribingData(
+          yauaaEnrichment.outputSchema,
+          json"""{"deviceClass":"Unknown"}"""
+        )
+      yauaaEnrichment.getYauaaContext("") shouldEqual defaultJson
+    }
   }
 
   /** Helper to check that a certain field of a parsed user agent has the expected value. */
@@ -239,26 +284,52 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers {
     "successfully construct a YauaaEnrichment case class with the right cache size if specified" in {
       val cacheSize = 42
 
-      val yauaaConfigJson = parse(s"""{
+      val yauaaConfigJson = json"""{
         "enabled": true,
         "parameters": {
           "cacheSize": $cacheSize
         }
-      }""").toOption.get
+      }"""
 
-      val expected = YauaaConf(schemaKey, Some(cacheSize))
+      val expected = YauaaConf(schemaKey, YauaaEnrichment.DefaultContextModel, Some(cacheSize))
       val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey)
       actual must beValid(expected)
     }
 
     "successfully construct a YauaaEnrichment case class with a default cache size if none specified" in {
-      val yauaaConfigJson = parse(s"""{
-        "enabled": true
-      }""").toOption.get
+      val yauaaConfigJson = json"""{"enabled": true }"""
 
-      val expected = YauaaConf(schemaKey, None)
+      val expected = YauaaConf(schemaKey, YauaaEnrichment.DefaultContextModel, None)
       val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey)
       actual must beValid(expected)
     }
+
+    "successfully construct a YauaaEnrichment case class with specific model" in {
+      val yauaaConfigJson = json"""{
+        "enabled": true,
+        "parameters": {
+          "cacheSize": 1,
+          "model": 2
+        }
+      }"""
+
+      val expected = YauaaConf(schemaKey, 2, Some(1))
+      val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey)
+      actual must beValid(expected)
+    }
+
+    "fail to construct a YauaaEnrichment case class with unsupported model" in {
+      val yauaaConfigJson = json"""{
+        "enabled": true,
+        "parameters": {
+          "cacheSize": 1,
+          "model": 3
+        }
+      }"""
+
+      val expected = NonEmptyList.one("YAUAA enrichment context with model 3 is not supported. Choose from: 1, 2")
+      val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey)
+      actual must beInvalid(expected)
+    }
   }
 }

project/Dependencies.scala

@@ -38,7 +38,7 @@ object Dependencies {
     val mysqlConnector   = "8.0.16"
     val jaywayJsonpath   = "2.4.0"
     val iabClient        = "0.2.0"
-    val yauaa            = "5.8"
+    val yauaa            = "5.19"
     val guava            = "28.1-jre"
     val slf4j            = "1.7.26"