From 5370576e7f70904cddff2f8e2d560a1a7867de0a Mon Sep 17 00:00:00 2001 From: Anton Parkhomenko Date: Wed, 26 Aug 2020 20:23:26 +0300 Subject: [PATCH] Common: bump YAUAA to 5.19 (close #314) --- .../enrichments/registry/EnrichmentConf.scala | 8 +- .../registry/YauaaEnrichment.scala | 46 +++++++-- .../enrichments/EnrichmentManagerSpec.scala | 2 +- .../registry/YauaaEnrichmentSpec.scala | 97 ++++++++++++++++--- project/Dependencies.scala | 2 +- 5 files changed, 129 insertions(+), 26 deletions(-) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala index 9e8743ce9..6f2efb7b3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala @@ -202,7 +202,11 @@ object EnrichmentConf { WeatherEnrichment[F](this) } - final case class YauaaConf(schemaKey: SchemaKey, cacheSize: Option[Int]) extends EnrichmentConf { - def enrichment: YauaaEnrichment = YauaaEnrichment(cacheSize) + final case class YauaaConf( + schemaKey: SchemaKey, + model: Int, + cacheSize: Option[Int] + ) extends EnrichmentConf { + def enrichment: YauaaEnrichment = YauaaEnrichment(model, cacheSize) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala index 1a3efc8c0..1d06358c8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala @@ -29,7 +29,7 @@ import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object to create an instance of YauaaEnrichment from the configuration. */ object YauaaEnrichment extends ParseableEnrichment { - val supportedSchema = + val supportedSchema: SchemaCriterion = SchemaCriterion( "com.snowplowanalytics.snowplow.enrichments", "yauaa_enrichment_config", @@ -38,6 +38,25 @@ object YauaaEnrichment extends ParseableEnrichment { 0 ) + /** + * Enrichment can produce `yauaa_context` with following models + * 1-0-0 comes from original YAUAA 5.8 + * 2-0-0 comes from YAUAA 5.19 and adds two new fields: + * `operatingSystemNameVersionMajor` and `operatingSystemVersionMajor` + * For 1-0-0 those fields are removed during context construction + */ + val SupportedContextModels = List(1, 2) + + /** Default `yauaa_context` MODEL from [[SupportedContextModels]] */ + val DefaultContextModel = 1 + + /** Fields that were added in YAUUA 5.19 and not supported by `iglu:nl.basjes/yauaa_context/1-0-0` */ + val Model2Fields: List[String] = + List(UserAgent.OPERATING_SYSTEM_NAME_VERSION_MAJOR, UserAgent.OPERATING_SYSTEM_VERSION_MAJOR) + + val DefaultDeviceClass = "Unknown" + val DefaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> DefaultDeviceClass) + /** * Creates a YauaaConf instance from a JValue containing the configuration of the enrichment. * @@ -54,7 +73,9 @@ object YauaaEnrichment extends ParseableEnrichment { (for { _ <- isParseable(c, schemaKey) cacheSize <- CirceUtils.extract[Option[Int]](c, "parameters", "cacheSize").toEither - } yield YauaaConf(schemaKey, cacheSize)).toValidatedNel + model <- CirceUtils.extract[Option[Int]](c, "parameters", "model").toEither.map(_.getOrElse(DefaultContextModel)) + _ <- checkModel(model) + } yield YauaaConf(schemaKey, model, cacheSize)).toValidatedNel /** Helper to decapitalize a string. Used for the names of the fields returned in the context. */ def decapitalize(s: String): String = @@ -63,14 +84,19 @@ object YauaaEnrichment extends ParseableEnrichment { case _ if s.length == 1 => s.toLowerCase case _ => s.charAt(0).toLower + s.substring(1) } + + private def checkModel(model: Int): Either[String, Unit] = + if (SupportedContextModels.contains(model)) ().asRight + else s"""YAUAA enrichment context with model $model is not supported. Choose from: ${SupportedContextModels.mkString(", ")}""".asLeft } /** * Class for YAUAA enrichment, which tries to parse and analyze the user agent string * and extract as many relevant attributes as possible, like for example the device class. + * @param model version of yauaa_context produced, `1` and `2` are supported currently * @param cacheSize Amount of user agents already parsed that stay in cache for faster parsing. */ -final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { +final case class YauaaEnrichment(model: Int, cacheSize: Option[Int]) extends Enrichment { import YauaaEnrichment.decapitalize private val uaa: UserAgentAnalyzer = { @@ -81,10 +107,7 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { a } - val outputSchema = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 0)) - - val defaultDeviceClass = "Unknown" - val defaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> defaultDeviceClass) + val outputSchema: SchemaKey = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(model, 0, 0)) /** * Gets the result of YAUAA user agent analysis as self-describing JSON, for a specific event. @@ -102,11 +125,16 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { def parseUserAgent(userAgent: String): Map[String, String] = userAgent match { case null | "" => - defaultResult + YauaaEnrichment.DefaultResult case _ => val parsedUA = uaa.parse(userAgent) - parsedUA.getAvailableFieldNames.asScala + parsedUA.getAvailableFieldNamesSorted.asScala + .filter(stripNewFields) .map(field => decapitalize(field) -> parsedUA.getValue(field)) .toMap } + + /** Predicate to remove YAUAA 5.19 fields if `iglu:nl.basjes/yauaa_context/1-0-0` is used */ + def stripNewFields(field: String): Boolean = + !(model == 1 && YauaaEnrichment.Model2Fields.contains(field)) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala index 83161af81..411d0b8f8 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala @@ -37,7 +37,7 @@ import enrichments.registry.JavascriptScriptEnrichment import enrichments.registry.YauaaEnrichment class EnrichmentManagerSpec extends Specification with EitherMatchers { - val enrichmentReg = EnrichmentRegistry[Id](yauaa = Some(YauaaEnrichment(None))) + val enrichmentReg = EnrichmentRegistry[Id](yauaa = Some(YauaaEnrichment(YauaaEnrichment.DefaultContextModel, None))) val client = SpecHelpers.client val processor = Processor("ssc-tests", "0.0.0") val timestamp = DateTime.now() diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala index 0e6f0e4af..aca3d74e2 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala @@ -12,11 +12,11 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry -import io.circe.parser._ +import cats.data.NonEmptyList + import io.circe.literal._ import nl.basjes.parse.useragent.UserAgent - import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.YauaaConf @@ -28,7 +28,11 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { import YauaaEnrichment.decapitalize - val yauaaEnrichment = YauaaEnrichment(None) + /** Default enrichment with 1-0-0 context */ + val yauaaEnrichment = YauaaEnrichment(YauaaEnrichment.DefaultContextModel, None) + + /** Enrichment with 2-0-0 context coming from YAUAA 5.19 */ + val yauaaEnrichment2 = YauaaEnrichment(2, None) // Devices val uaIpad = @@ -68,11 +72,11 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "YAUAA enrichment should" >> { "return default value for null" >> { - yauaaEnrichment.parseUserAgent(null) shouldEqual yauaaEnrichment.defaultResult + yauaaEnrichment.parseUserAgent(null) shouldEqual YauaaEnrichment.DefaultResult } "return default value for empty user agent" >> { - yauaaEnrichment.parseUserAgent("") shouldEqual yauaaEnrichment.defaultResult + yauaaEnrichment.parseUserAgent("") shouldEqual YauaaEnrichment.DefaultResult } "detect correctly DeviceClass" >> { @@ -185,7 +189,7 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { ) } - "create a JSON with the schema and the data" >> { + "create a JSON with the schema 1-0-0 and the data" >> { val expected = SelfDescribingData( yauaaEnrichment.outputSchema, @@ -201,6 +205,47 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { ) yauaaEnrichment.getYauaaContext("") shouldEqual defaultJson } + + "create a JSON with the schema 2-0-0 and the data" >> { + val expected = + SelfDescribingData( + SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(2, 0, 0)), + json"""{ + "deviceBrand":"Samsung", + "deviceName":"Samsung SM-G960F", + "layoutEngineNameVersion":"Blink 62.0", + "operatingSystemNameVersion":"Android 8.0.0", + "operatingSystemVersionBuild":"R16NW", + "layoutEngineNameVersionMajor":"Blink 62", + "operatingSystemName":"Android", + "agentVersionMajor":"62", + "layoutEngineVersionMajor":"62", + "deviceClass":"Phone", + "agentNameVersionMajor":"Chrome 62", + "operatingSystemClass":"Mobile", + "layoutEngineName":"Blink", + "agentName":"Chrome", + "agentVersion":"62.0.3202.84", + "layoutEngineClass":"Browser", + "agentNameVersion":"Chrome 62.0.3202.84", + "operatingSystemVersion":"8.0.0", + "agentClass":"Browser", + "layoutEngineVersion":"62.0", + + "operatingSystemVersionMajor" : "8", + "operatingSystemNameVersionMajor" : "Android 8" + }""" + ) + val actual = yauaaEnrichment2.getYauaaContext(uaGalaxyS9) + actual shouldEqual expected + + val defaultJson = + SelfDescribingData( + yauaaEnrichment.outputSchema, + json"""{"deviceClass":"Unknown"}""" + ) + yauaaEnrichment.getYauaaContext("") shouldEqual defaultJson + } } /** Helper to check that a certain field of a parsed user agent has the expected value. */ @@ -239,26 +284,52 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "successfully construct a YauaaEnrichment case class with the right cache size if specified" in { val cacheSize = 42 - val yauaaConfigJson = parse(s"""{ + val yauaaConfigJson = json"""{ "enabled": true, "parameters": { "cacheSize": $cacheSize } - }""").toOption.get + }""" - val expected = YauaaConf(schemaKey, Some(cacheSize)) + val expected = YauaaConf(schemaKey, YauaaEnrichment.DefaultContextModel, Some(cacheSize)) val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey) actual must beValid(expected) } "successfully construct a YauaaEnrichment case class with a default cache size if none specified" in { - val yauaaConfigJson = parse(s"""{ - "enabled": true - }""").toOption.get + val yauaaConfigJson = json"""{"enabled": true }""" - val expected = YauaaConf(schemaKey, None) + val expected = YauaaConf(schemaKey, YauaaEnrichment.DefaultContextModel, None) val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey) actual must beValid(expected) } + + "successfully construct a YauaaEnrichment case class with specific model" in { + val yauaaConfigJson = json"""{ + "enabled": true, + "parameters": { + "cacheSize": 1, + "model": 2 + } + }""" + + val expected = YauaaConf(schemaKey, 2, Some(1)) + val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey) + actual must beValid(expected) + } + + "fail to construct a YauaaEnrichment case class with unsupported model" in { + val yauaaConfigJson = json"""{ + "enabled": true, + "parameters": { + "cacheSize": 1, + "model": 3 + } + }""" + + val expected = NonEmptyList.one("YAUAA enrichment context with model 3 is not supported. Choose from: 1, 2") + val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey) + actual must beInvalid(expected) + } } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d989ca263..803b8d242 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -38,7 +38,7 @@ object Dependencies { val mysqlConnector = "8.0.16" val jaywayJsonpath = "2.4.0" val iabClient = "0.2.0" - val yauaa = "5.8" + val yauaa = "5.19" val guava = "28.1-jre" val slf4j = "1.7.26"