From ae1b94f4b06dc13fa0ed5b7fc452d5eca0c16c6f Mon Sep 17 00:00:00 2001 From: Anton Parkhomenko Date: Wed, 16 Sep 2020 20:50:32 +0300 Subject: [PATCH] Common: bump YAUAA to 5.19 (close #314) --- .../enrichments/YauaaEnrichmentSpec.scala | 3 +- .../enrichments/registry/EnrichmentConf.scala | 5 +- .../registry/YauaaEnrichment.scala | 18 +++---- .../registry/YauaaEnrichmentSpec.scala | 52 +++++++++++++------ project/Dependencies.scala | 2 +- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala index 1f938b4e5..1610c046b 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala @@ -36,8 +36,7 @@ object YauaaEnrichmentSpec { "event_format" -> "jsonschema", "event_version" -> "1-0-0", "event" -> "page_ping", - "derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-0","data":{"deviceBrand":"Unknown","deviceName":"Desktop","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces - ) + "derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-1","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces) } class YauaaEnrichmentSpec extends PipelineSpec { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala index 9e8743ce9..4502ea59d 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala @@ -202,7 +202,10 @@ object EnrichmentConf { WeatherEnrichment[F](this) } - final case class YauaaConf(schemaKey: SchemaKey, cacheSize: Option[Int]) extends EnrichmentConf { + final case class YauaaConf( + schemaKey: SchemaKey, + cacheSize: Option[Int] + ) extends EnrichmentConf { def enrichment: YauaaEnrichment = YauaaEnrichment(cacheSize) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala index 1a3efc8c0..a33406fb1 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala @@ -29,7 +29,7 @@ import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object to create an instance of YauaaEnrichment from the configuration. */ object YauaaEnrichment extends ParseableEnrichment { - val supportedSchema = + val supportedSchema: SchemaCriterion = SchemaCriterion( "com.snowplowanalytics.snowplow.enrichments", "yauaa_enrichment_config", @@ -38,6 +38,11 @@ object YauaaEnrichment extends ParseableEnrichment { 0 ) + val DefaultDeviceClass = "Unknown" + val DefaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> DefaultDeviceClass) + + val outputSchema: SchemaKey = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 1)) + /** * Creates a YauaaConf instance from a JValue containing the configuration of the enrichment. * @@ -81,18 +86,13 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { a } - val outputSchema = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 0)) - - val defaultDeviceClass = "Unknown" - val defaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> defaultDeviceClass) - /** * Gets the result of YAUAA user agent analysis as self-describing JSON, for a specific event. * @param userAgent User agent of the event. * @return Attributes retrieved thanks to the user agent (if any), as self-describing JSON. */ def getYauaaContext(userAgent: String): SelfDescribingData[Json] = - SelfDescribingData(outputSchema, parseUserAgent(userAgent).asJson) + SelfDescribingData(YauaaEnrichment.outputSchema, parseUserAgent(userAgent).asJson) /** * Gets the map of attributes retrieved by YAUAA from the user agent. @@ -102,10 +102,10 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { def parseUserAgent(userAgent: String): Map[String, String] = userAgent match { case null | "" => - defaultResult + YauaaEnrichment.DefaultResult case _ => val parsedUA = uaa.parse(userAgent) - parsedUA.getAvailableFieldNames.asScala + parsedUA.getAvailableFieldNamesSorted.asScala .map(field => decapitalize(field) -> parsedUA.getValue(field)) .toMap } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala index 0e6f0e4af..fef14842f 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala @@ -12,15 +12,11 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry -import io.circe.parser._ import io.circe.literal._ import nl.basjes.parse.useragent.UserAgent - import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} -import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.YauaaConf - import org.specs2.matcher.ValidatedMatchers import org.specs2.mutable.Specification @@ -28,6 +24,7 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { import YauaaEnrichment.decapitalize + /** Default enrichment with 1-0-0 context */ val yauaaEnrichment = YauaaEnrichment(None) // Devices @@ -68,11 +65,11 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "YAUAA enrichment should" >> { "return default value for null" >> { - yauaaEnrichment.parseUserAgent(null) shouldEqual yauaaEnrichment.defaultResult + yauaaEnrichment.parseUserAgent(null) shouldEqual YauaaEnrichment.DefaultResult } "return default value for empty user agent" >> { - yauaaEnrichment.parseUserAgent("") shouldEqual yauaaEnrichment.defaultResult + yauaaEnrichment.parseUserAgent("") shouldEqual YauaaEnrichment.DefaultResult } "detect correctly DeviceClass" >> { @@ -185,18 +182,41 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { ) } - "create a JSON with the schema and the data" >> { + "create a JSON with the schema 1-0-0 and the data" >> { val expected = SelfDescribingData( - yauaaEnrichment.outputSchema, - json"""{"deviceBrand":"Samsung","deviceName":"Samsung SM-G960F","layoutEngineNameVersion":"Blink 62.0","operatingSystemNameVersion":"Android 8.0.0","operatingSystemVersionBuild":"R16NW","layoutEngineNameVersionMajor":"Blink 62","operatingSystemName":"Android","agentVersionMajor":"62","layoutEngineVersionMajor":"62","deviceClass":"Phone","agentNameVersionMajor":"Chrome 62","operatingSystemClass":"Mobile","layoutEngineName":"Blink","agentName":"Chrome","agentVersion":"62.0.3202.84","layoutEngineClass":"Browser","agentNameVersion":"Chrome 62.0.3202.84","operatingSystemVersion":"8.0.0","agentClass":"Browser","layoutEngineVersion":"62.0"}""" + YauaaEnrichment.outputSchema, + json"""{ + "deviceBrand":"Samsung", + "deviceName":"Samsung SM-G960F", + "layoutEngineNameVersion":"Blink 62.0", + "operatingSystemNameVersion":"Android 8.0.0", + "operatingSystemVersionBuild":"R16NW", + "layoutEngineNameVersionMajor":"Blink 62", + "operatingSystemName":"Android", + "agentVersionMajor":"62", + "layoutEngineVersionMajor":"62", + "deviceClass":"Phone", + "agentNameVersionMajor":"Chrome 62", + "operatingSystemClass":"Mobile", + "layoutEngineName":"Blink", + "agentName":"Chrome", + "agentVersion":"62.0.3202.84", + "layoutEngineClass":"Browser", + "agentNameVersion":"Chrome 62.0.3202.84", + "operatingSystemVersion":"8.0.0", + "agentClass":"Browser", + "layoutEngineVersion":"62.0", + "operatingSystemNameVersionMajor":"Android 8", + "operatingSystemVersionMajor":"8" + }""" ) val actual = yauaaEnrichment.getYauaaContext(uaGalaxyS9) actual shouldEqual expected val defaultJson = SelfDescribingData( - yauaaEnrichment.outputSchema, + YauaaEnrichment.outputSchema, json"""{"deviceClass":"Unknown"}""" ) yauaaEnrichment.getYauaaContext("") shouldEqual defaultJson @@ -239,24 +259,22 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "successfully construct a YauaaEnrichment case class with the right cache size if specified" in { val cacheSize = 42 - val yauaaConfigJson = parse(s"""{ + val yauaaConfigJson = json"""{ "enabled": true, "parameters": { "cacheSize": $cacheSize } - }""").toOption.get + }""" - val expected = YauaaConf(schemaKey, Some(cacheSize)) + val expected = EnrichmentConf.YauaaConf(schemaKey, Some(cacheSize)) val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey) actual must beValid(expected) } "successfully construct a YauaaEnrichment case class with a default cache size if none specified" in { - val yauaaConfigJson = parse(s"""{ - "enabled": true - }""").toOption.get + val yauaaConfigJson = json"""{"enabled": true }""" - val expected = YauaaConf(schemaKey, None) + val expected = EnrichmentConf.YauaaConf(schemaKey, None) val actual = YauaaEnrichment.parse(yauaaConfigJson, schemaKey) actual must beValid(expected) } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d989ca263..803b8d242 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -38,7 +38,7 @@ object Dependencies { val mysqlConnector = "8.0.16" val jaywayJsonpath = "2.4.0" val iabClient = "0.2.0" - val yauaa = "5.8" + val yauaa = "5.19" val guava = "28.1-jre" val slf4j = "1.7.26"