Common: bump snowplow-badrows to 2.1.0 (close #325)

snowplow · Sep 17, 2020 · 873f143 · 873f143
1 parent ae1b94f
commit 873f143
Show file tree

Hide file tree

Showing 10 changed files with 48 additions and 35 deletions.
diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala
@@ -171,7 +171,7 @@ object SpecHelpers {
 
   def copyResource(resource: String, localFile: String): Unit = {
     Files.copy(
-      Paths.get(getClass.getResource(resource).toURI()),
+      Paths.get(getClass.getResource(resource).toURI),
       Paths.get(localFile)
     )
     ()
@@ -180,7 +180,7 @@ object SpecHelpers {
 
 object CI
     extends Tag(
-      if (sys.env.get("CI").map(_ == "true").getOrElse(false)) "" else classOf[Ignore].getName
+      if (sys.env.get("CI").contains("true")) "" else classOf[Ignore].getName
     )
 
-object OER extends Tag(if (sys.env.get("OER_KEY").isDefined) "" else classOf[Ignore].getName)
+object OER extends Tag(if (sys.env.contains("OER_KEY")) "" else classOf[Ignore].getName)
diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/UtilsSpec.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/UtilsSpec.scala
@@ -19,10 +19,13 @@ import java.time.Instant
 
 import com.snowplowanalytics.snowplow.badrows._
 import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent
+
 import cats.implicits._
+
 import io.circe.parser
+
 import com.snowplowanalytics.iglu.core.SelfDescribingData
-import com.snowplowanalytics.iglu.core.circe.instances._
+import com.snowplowanalytics.iglu.core.circe.implicits._
 
 import utils._
 import org.scalatest.freespec.AnyFreeSpec
@@ -111,7 +114,7 @@ class UtilsSpec extends AnyFreeSpec with Matchers {
         badRowSizeViolation.failure.maximumAllowedSizeBytes shouldEqual 150
         badRowSizeViolation.failure.actualSizeBytes shouldEqual 267
         badRowSizeViolation.failure.expectation shouldEqual "bad row exceeded the maximum size"
-        badRowSizeViolation.payload.line shouldEqual "{\"schema\":\"iglu"
+        badRowSizeViolation.payload.event shouldEqual "{\"schema\":\"iglu"
         badRowSizeViolation.processor shouldEqual processor
       }
     }
@@ -124,7 +127,7 @@ class UtilsSpec extends AnyFreeSpec with Matchers {
         badRowSizeViolation.failure.maximumAllowedSizeBytes shouldEqual 400
         badRowSizeViolation.failure.actualSizeBytes shouldEqual 100
         badRowSizeViolation.failure.expectation shouldEqual "event passed enrichment but exceeded the maximum allowed size as a result"
-        badRowSizeViolation.payload.line shouldEqual ("a" * 40)
+        badRowSizeViolation.payload.event shouldEqual ("a" * 40)
         badRowSizeViolation.processor shouldEqual processor
       }
     }

diff --git a/...in/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala b/...in/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala
@@ -10,26 +10,29 @@
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
  */
-package com.snowplowanalytics.snowplow.enrich.common
-package adapters
-package registry
+package com.snowplowanalytics.snowplow.enrich.common.adapters.registry
 
 import cats.Monad
 import cats.data.{NonEmptyList, ValidatedNel}
 import cats.effect.Clock
 import cats.syntax.either._
 import cats.syntax.option._
 import cats.syntax.validated._
+
+import com.snowplowanalytics.iglu.core.{SchemaKey, SelfDescribingData}
+import com.snowplowanalytics.iglu.core.circe.implicits._
+
 import com.snowplowanalytics.iglu.client.Client
 import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup
-import com.snowplowanalytics.iglu.core.{SchemaKey, SelfDescribingData}
-import com.snowplowanalytics.iglu.core.circe.instances._
+
 import com.snowplowanalytics.snowplow.badrows._
+
 import io.circe._
 import io.circe.syntax._
 
-import loaders.CollectorPayload
-import utils.{ConversionUtils, HttpClient, JsonUtils}
+import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent
+import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload
+import com.snowplowanalytics.snowplow.enrich.common.utils.{ConversionUtils, HttpClient, JsonUtils}
 
 /**
  * Transforms a collector payload which either:

diff --git a/.../com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala b/.../com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala
@@ -24,7 +24,7 @@ import io.circe.Json
 import com.snowplowanalytics.iglu.client.Client
 import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup
 import com.snowplowanalytics.iglu.core.{SchemaCriterion, SelfDescribingData}
-import com.snowplowanalytics.iglu.core.circe.instances._
+import com.snowplowanalytics.iglu.core.circe.implicits._
 
 import com.snowplowanalytics.snowplow.badrows.FailureDetails
 
@@ -41,9 +41,9 @@ import com.snowplowanalytics.snowplow.enrich.common.utils.{HttpClient, JsonUtils
 object Tp2Adapter extends Adapter {
   // Expected content types for a request body
   private object ContentTypes {
-    val list =
+    val list: List[String] =
       List("application/json", "application/json; charset=utf-8", "application/json; charset=UTF-8")
-    val str = list.mkString(", ")
+    val str: String = list.mkString(", ")
   }
 
   // Request body expected to validate against this JSON Schema
@@ -105,7 +105,7 @@ object Tp2Adapter extends Adapter {
         case (None, None) => Monad[F].pure(NonEmptyList.one(qsParams).valid)
         case (Some(bdy), Some(_)) => // Build our NEL of parameters
           (for {
-            json <- extractAndValidateJson(PayloadDataSchema, bdy, "body", client)
+            json <- extractAndValidateJson(PayloadDataSchema, bdy, client)
             nel <- EitherT.fromEither[F](toParametersNel(json, qsParams))
           } yield nel).toValidated
       }
@@ -201,7 +201,6 @@ object Tp2Adapter extends Adapter {
 
   /**
    * Extract the JSON from a String, and validate it against the supplied JSON Schema.
-   * @param field The name of the field containing the JSON instance
    * @param schemaCriterion The schema that we expected this self-describing JSON to conform to
    * @param instance A JSON instance as String
    * @param client Our Iglu client, for schema lookups
@@ -211,7 +210,6 @@ object Tp2Adapter extends Adapter {
   private def extractAndValidateJson[F[_]: Monad: RegistryLookup: Clock](
     schemaCriterion: SchemaCriterion,
     instance: String,
-    field: String,
     client: Client[F, Json]
   ): EitherT[F, NonEmptyList[FailureDetails.TrackerProtocolViolation], Json] =
     (for {
@@ -220,7 +218,7 @@ object Tp2Adapter extends Adapter {
                .leftMap(e =>
                  NonEmptyList.one(
                    FailureDetails.TrackerProtocolViolation
-                     .NotJson(field, instance.some, e)
+                     .NotJson("body", instance.some, e)
                  )
                )
            )

diff --git a/...n/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala b/...n/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala
@@ -22,7 +22,7 @@ import io.circe._
 import io.circe.syntax._
 
 import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData}
-import com.snowplowanalytics.iglu.core.circe.instances._
+import com.snowplowanalytics.iglu.core.circe.implicits._
 
 import com.snowplowanalytics.iglu.client.Client
 import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup

diff --git a/...alytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala b/...alytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala
@@ -147,7 +147,6 @@ final case class ApiRequestEnrichment[F[_]: Monad: HttpClient](
    * @return validated list of lookups, whole lookup will be failed if any of outputs were failed
    */
   private[apirequest] def getOutputs(validInputs: Option[Map[String, String]]): EitherT[F, NonEmptyList[String], List[Json]] = {
-    import cats.instances.parallel._
     val result: List[F[Either[Throwable, Json]]] =
       for {
         templateContext <- validInputs.toList

diff --git a/.../common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala b/.../common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala
@@ -24,10 +24,9 @@ import java.time.Instant
 import com.snowplowanalytics.iglu.client.{Client, ClientError}
 import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup
 
-import com.snowplowanalytics.iglu.core.circe.instances._
 import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData}
+import com.snowplowanalytics.iglu.core.circe.implicits._
 
-import com.snowplowanalytics.snowplow.badrows.FailureDetails
 import com.snowplowanalytics.snowplow.badrows._
 
 import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent
@@ -281,7 +280,7 @@ object IgluUtils {
     pee: Payload.PartiallyEnrichedEvent,
     re: Payload.RawEvent,
     processor: Processor
-  ) =
+  ): BadRow.SchemaViolations =
     BadRow.SchemaViolations(
       processor,
       Failure.SchemaViolations(Instant.now(), vs),

diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala
@@ -14,14 +14,19 @@ package com.snowplowanalytics.snowplow.enrich.common
 
 import cats.Id
 import cats.implicits._
+
 import com.snowplowanalytics.iglu.client.Client
 import com.snowplowanalytics.iglu.core.SelfDescribingData
-import com.snowplowanalytics.iglu.core.circe.instances._
+import com.snowplowanalytics.iglu.core.circe.implicits._
+
 import com.snowplowanalytics.lrumap.CreateLruMap._
+
 import io.circe.Json
 import io.circe.literal._
+
 import org.apache.http.NameValuePair
 import org.apache.http.message.BasicNameValuePair
+
 import com.snowplowanalytics.snowplow.enrich.common.utils.JsonUtils
 
 object SpecHelpers {

diff --git a/...com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala b/...com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala
@@ -10,21 +10,26 @@
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
  */
-package com.snowplowanalytics.snowplow.enrich.common
-package adapters
-package registry
+package com.snowplowanalytics.snowplow.enrich.common.adapters.registry
 
 import cats.data.NonEmptyList
 import cats.syntax.option._
+
 import com.snowplowanalytics.snowplow.badrows._
+
 import io.circe._
 import io.circe.literal._
+
 import org.joda.time.DateTime
+
 import org.specs2.Specification
 import org.specs2.matcher.{DataTables, ValidatedMatchers}
 
-import loaders._
-import utils.Clock._
+import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent
+import com.snowplowanalytics.snowplow.enrich.common.loaders._
+import com.snowplowanalytics.snowplow.enrich.common.utils.Clock._
+
+import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers
 
 class MailchimpAdapterSpec extends Specification with DataTables with ValidatedMatchers {
   def is = s2"""

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -44,15 +44,15 @@ object Dependencies {
 
     val refererParser    = "1.0.0"
     val maxmindIplookups = "0.6.1"
-    val circe            = "0.11.1"
-    val circeOptics      = "0.11.0"
-    val circeJackson     = "0.11.1"
+    val circe            = "0.13.0"
+    val circeOptics      = "0.13.0"
+    val circeJackson     = "0.13.0"
     val scalaForex       = "0.7.0"
     val scalaWeather     = "0.5.0"
     val gatlingJsonpath  = "0.6.14"
     val scalaUri         = "1.4.5"
     val scalaLruMap      = "0.3.1"
-    val badRows          = "1.0.0"
+    val badRows          = "2.1.0"
 
     val snowplowRawEvent = "0.1.0"
     val collectorPayload = "0.0.0"
@@ -98,6 +98,7 @@ object Dependencies {
     val circeParser      = "io.circe"                   %% "circe-parser"                  % V.circe
     val circeLiteral     = "io.circe"                   %% "circe-literal"                 % V.circe
     val circeJava8       = "io.circe"                   %% "circe-java8"                   % V.circe
+    val circeJawn        = "io.circe"                   %% "circe-jawn"                    % V.circe
     val circeOptics      = "io.circe"                   %% "circe-optics"                  % V.circeOptics
     val circeJackson     = "io.circe"                   %% "circe-jackson29"               % V.circeJackson
     val scalaUri         = "io.lemonlabs"               %% "scala-uri"                     % V.scalaUri