From 6f925d56065944e580cc2362597e26bb332b67b0 Mon Sep 17 00:00:00 2001 From: Anton Parkhomenko Date: Wed, 26 Aug 2020 20:55:04 +0300 Subject: [PATCH] Common: bump snowplow-badrows to 2.1.0 (close #325) --- .../SpecHelpers.scala | 6 +++--- .../UtilsSpec.scala | 9 ++++++--- .../common/adapters/registry/IgluAdapter.scala | 17 ++++++++++------- .../adapters/registry/snowplow/Tp2Adapter.scala | 12 +++++------- .../common/enrichments/EnrichmentRegistry.scala | 2 +- .../apirequest/ApiRequestEnrichment.scala | 1 - .../common/utils/IgluUtils.scala | 5 ++--- .../SpecHelpers.scala | 7 ++++++- .../registry/MailchimpAdapterSpec.scala | 15 ++++++++++----- project/Dependencies.scala | 9 +++++---- 10 files changed, 48 insertions(+), 35 deletions(-) diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala index a0a88a2ef..d7914cebc 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala @@ -171,7 +171,7 @@ object SpecHelpers { def copyResource(resource: String, localFile: String): Unit = { Files.copy( - Paths.get(getClass.getResource(resource).toURI()), + Paths.get(getClass.getResource(resource).toURI), Paths.get(localFile) ) () @@ -180,7 +180,7 @@ object SpecHelpers { object CI extends Tag( - if (sys.env.get("CI").map(_ == "true").getOrElse(false)) "" else classOf[Ignore].getName + if (sys.env.get("CI").contains("true")) "" else classOf[Ignore].getName ) -object OER extends Tag(if (sys.env.get("OER_KEY").isDefined) "" else classOf[Ignore].getName) +object OER extends Tag(if (sys.env.contains("OER_KEY")) "" else classOf[Ignore].getName) diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/UtilsSpec.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/UtilsSpec.scala index ec81ba929..76ce5ce88 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/UtilsSpec.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/UtilsSpec.scala @@ -19,10 +19,13 @@ import java.time.Instant import com.snowplowanalytics.snowplow.badrows._ import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent + import cats.implicits._ + import io.circe.parser + import com.snowplowanalytics.iglu.core.SelfDescribingData -import com.snowplowanalytics.iglu.core.circe.instances._ +import com.snowplowanalytics.iglu.core.circe.implicits._ import utils._ import org.scalatest.freespec.AnyFreeSpec @@ -111,7 +114,7 @@ class UtilsSpec extends AnyFreeSpec with Matchers { badRowSizeViolation.failure.maximumAllowedSizeBytes shouldEqual 150 badRowSizeViolation.failure.actualSizeBytes shouldEqual 267 badRowSizeViolation.failure.expectation shouldEqual "bad row exceeded the maximum size" - badRowSizeViolation.payload.line shouldEqual "{\"schema\":\"iglu" + badRowSizeViolation.payload.event shouldEqual "{\"schema\":\"iglu" badRowSizeViolation.processor shouldEqual processor } } @@ -124,7 +127,7 @@ class UtilsSpec extends AnyFreeSpec with Matchers { badRowSizeViolation.failure.maximumAllowedSizeBytes shouldEqual 400 badRowSizeViolation.failure.actualSizeBytes shouldEqual 100 badRowSizeViolation.failure.expectation shouldEqual "event passed enrichment but exceeded the maximum allowed size as a result" - badRowSizeViolation.payload.line shouldEqual ("a" * 40) + badRowSizeViolation.payload.event shouldEqual ("a" * 40) badRowSizeViolation.processor shouldEqual processor } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala index cb6de3554..4c5c9560b 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala @@ -10,9 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package adapters -package registry +package com.snowplowanalytics.snowplow.enrich.common.adapters.registry import cats.Monad import cats.data.{NonEmptyList, ValidatedNel} @@ -20,16 +18,21 @@ import cats.effect.Clock import cats.syntax.either._ import cats.syntax.option._ import cats.syntax.validated._ + +import com.snowplowanalytics.iglu.core.{SchemaKey, SelfDescribingData} +import com.snowplowanalytics.iglu.core.circe.implicits._ + import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup -import com.snowplowanalytics.iglu.core.{SchemaKey, SelfDescribingData} -import com.snowplowanalytics.iglu.core.circe.instances._ + import com.snowplowanalytics.snowplow.badrows._ + import io.circe._ import io.circe.syntax._ -import loaders.CollectorPayload -import utils.{ConversionUtils, HttpClient, JsonUtils} +import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent +import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload +import com.snowplowanalytics.snowplow.enrich.common.utils.{ConversionUtils, HttpClient, JsonUtils} /** * Transforms a collector payload which either: diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala index 10051a596..6291851fb 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala @@ -24,7 +24,7 @@ import io.circe.Json import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup import com.snowplowanalytics.iglu.core.{SchemaCriterion, SelfDescribingData} -import com.snowplowanalytics.iglu.core.circe.instances._ +import com.snowplowanalytics.iglu.core.circe.implicits._ import com.snowplowanalytics.snowplow.badrows.FailureDetails @@ -41,9 +41,9 @@ import com.snowplowanalytics.snowplow.enrich.common.utils.{HttpClient, JsonUtils object Tp2Adapter extends Adapter { // Expected content types for a request body private object ContentTypes { - val list = + val list: List[String] = List("application/json", "application/json; charset=utf-8", "application/json; charset=UTF-8") - val str = list.mkString(", ") + val str: String = list.mkString(", ") } // Request body expected to validate against this JSON Schema @@ -105,7 +105,7 @@ object Tp2Adapter extends Adapter { case (None, None) => Monad[F].pure(NonEmptyList.one(qsParams).valid) case (Some(bdy), Some(_)) => // Build our NEL of parameters (for { - json <- extractAndValidateJson(PayloadDataSchema, bdy, "body", client) + json <- extractAndValidateJson(PayloadDataSchema, bdy, client) nel <- EitherT.fromEither[F](toParametersNel(json, qsParams)) } yield nel).toValidated } @@ -201,7 +201,6 @@ object Tp2Adapter extends Adapter { /** * Extract the JSON from a String, and validate it against the supplied JSON Schema. - * @param field The name of the field containing the JSON instance * @param schemaCriterion The schema that we expected this self-describing JSON to conform to * @param instance A JSON instance as String * @param client Our Iglu client, for schema lookups @@ -211,7 +210,6 @@ object Tp2Adapter extends Adapter { private def extractAndValidateJson[F[_]: Monad: RegistryLookup: Clock]( schemaCriterion: SchemaCriterion, instance: String, - field: String, client: Client[F, Json] ): EitherT[F, NonEmptyList[FailureDetails.TrackerProtocolViolation], Json] = (for { @@ -220,7 +218,7 @@ object Tp2Adapter extends Adapter { .leftMap(e => NonEmptyList.one( FailureDetails.TrackerProtocolViolation - .NotJson(field, instance.some, e) + .NotJson("body", instance.some, e) ) ) ) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala index 015e3f5c8..fa6e36ef1 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala @@ -22,7 +22,7 @@ import io.circe._ import io.circe.syntax._ import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} -import com.snowplowanalytics.iglu.core.circe.instances._ +import com.snowplowanalytics.iglu.core.circe.implicits._ import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala index fd589350a..9b9ed39b8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala @@ -147,7 +147,6 @@ final case class ApiRequestEnrichment[F[_]: Monad: HttpClient]( * @return validated list of lookups, whole lookup will be failed if any of outputs were failed */ private[apirequest] def getOutputs(validInputs: Option[Map[String, String]]): EitherT[F, NonEmptyList[String], List[Json]] = { - import cats.instances.parallel._ val result: List[F[Either[Throwable, Json]]] = for { templateContext <- validInputs.toList diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala index e8c1db21d..b445c1a7c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala @@ -24,10 +24,9 @@ import java.time.Instant import com.snowplowanalytics.iglu.client.{Client, ClientError} import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup -import com.snowplowanalytics.iglu.core.circe.instances._ import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} +import com.snowplowanalytics.iglu.core.circe.implicits._ -import com.snowplowanalytics.snowplow.badrows.FailureDetails import com.snowplowanalytics.snowplow.badrows._ import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent @@ -281,7 +280,7 @@ object IgluUtils { pee: Payload.PartiallyEnrichedEvent, re: Payload.RawEvent, processor: Processor - ) = + ): BadRow.SchemaViolations = BadRow.SchemaViolations( processor, Failure.SchemaViolations(Instant.now(), vs), diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala index d0f147d5d..24e73c7bb 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala @@ -14,14 +14,19 @@ package com.snowplowanalytics.snowplow.enrich.common import cats.Id import cats.implicits._ + import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.iglu.core.SelfDescribingData -import com.snowplowanalytics.iglu.core.circe.instances._ +import com.snowplowanalytics.iglu.core.circe.implicits._ + import com.snowplowanalytics.lrumap.CreateLruMap._ + import io.circe.Json import io.circe.literal._ + import org.apache.http.NameValuePair import org.apache.http.message.BasicNameValuePair + import com.snowplowanalytics.snowplow.enrich.common.utils.JsonUtils object SpecHelpers { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala index 88c92c9e2..990b33f79 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala @@ -10,21 +10,26 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package adapters -package registry +package com.snowplowanalytics.snowplow.enrich.common.adapters.registry import cats.data.NonEmptyList import cats.syntax.option._ + import com.snowplowanalytics.snowplow.badrows._ + import io.circe._ import io.circe.literal._ + import org.joda.time.DateTime + import org.specs2.Specification import org.specs2.matcher.{DataTables, ValidatedMatchers} -import loaders._ -import utils.Clock._ +import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent +import com.snowplowanalytics.snowplow.enrich.common.loaders._ +import com.snowplowanalytics.snowplow.enrich.common.utils.Clock._ + +import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers class MailchimpAdapterSpec extends Specification with DataTables with ValidatedMatchers { def is = s2""" diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 803b8d242..64dc9f7dd 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -44,15 +44,15 @@ object Dependencies { val refererParser = "1.0.0" val maxmindIplookups = "0.6.1" - val circe = "0.11.1" - val circeOptics = "0.11.0" - val circeJackson = "0.11.1" + val circe = "0.13.0" + val circeOptics = "0.13.0" + val circeJackson = "0.13.0" val scalaForex = "0.7.0" val scalaWeather = "0.5.0" val gatlingJsonpath = "0.6.14" val scalaUri = "1.4.5" val scalaLruMap = "0.3.1" - val badRows = "1.0.0" + val badRows = "2.1.0" val snowplowRawEvent = "0.1.0" val collectorPayload = "0.0.0" @@ -98,6 +98,7 @@ object Dependencies { val circeParser = "io.circe" %% "circe-parser" % V.circe val circeLiteral = "io.circe" %% "circe-literal" % V.circe val circeJava8 = "io.circe" %% "circe-java8" % V.circe + val circeJawn = "io.circe" %% "circe-jawn" % V.circe val circeOptics = "io.circe" %% "circe-optics" % V.circeOptics val circeJackson = "io.circe" %% "circe-jackson29" % V.circeJackson val scalaUri = "io.lemonlabs" %% "scala-uri" % V.scalaUri