diff --git a/modules/beam/src/main/scala/com.snowplowanalytics.snowplow.enrich.beam/utils.scala b/modules/beam/src/main/scala/com.snowplowanalytics.snowplow.enrich.beam/utils.scala index 6b07de9e9..1c4ff1d02 100644 --- a/modules/beam/src/main/scala/com.snowplowanalytics.snowplow.enrich.beam/utils.scala +++ b/modules/beam/src/main/scala/com.snowplowanalytics.snowplow.enrich.beam/utils.scala @@ -25,14 +25,18 @@ import scala.util.Try import cats.Id import cats.effect.Clock + import io.circe.Json import io.circe.syntax._ -import com.snowplowanalytics.snowplow.badrows._ -import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent + import org.joda.time.{DateTime, DateTimeZone} import org.joda.time.format.DateTimeFormat + +import com.snowplowanalytics.snowplow.badrows._ + +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf -import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.PiiPseudonymizerConf +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.PiiPseudonymizerConf object utils { @@ -88,10 +92,8 @@ object utils { /** Determine if we have to emit pii transformation events. */ def emitPii(confs: List[EnrichmentConf]): Boolean = confs - .collect { case c: PiiPseudonymizerConf => c } - .headOption - .map(_.emitIdentificationEvent) - .getOrElse(false) + .collectFirst { case c: PiiPseudonymizerConf => c } + .exists(_.emitIdentificationEvent) // We want to take one-tenth of the payload characters (not taking into account multi-bytes char) private val ReductionFactor = 10 diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SingletonSpec.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SingletonSpec.scala index 4f72161ce..2a250baa3 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SingletonSpec.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SingletonSpec.scala @@ -14,14 +14,16 @@ */ package com.snowplowanalytics.snowplow.enrich.beam -import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry._ import io.circe.literal._ -import org.scalatest._ -import matchers.should.Matchers._ -import singleton._ +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry._ +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.AnonIpConf + +import org.scalatest.matchers.should.Matchers._ import org.scalatest.freespec.AnyFreeSpec +import com.snowplowanalytics.snowplow.enrich.beam.singleton._ + class SingletonSpec extends AnyFreeSpec { "the singleton object should" - { "make a ClientSingleton.get function available" - { diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala index 5e8f249cc..a0a88a2ef 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/SpecHelpers.scala @@ -114,7 +114,7 @@ object SpecHelpers { contentType: Option[String] = None, headers: List[String] = Nil, ipAddress: String = "", - networkUserId: String = java.util.UUID.randomUUID().toString(), + networkUserId: String = java.util.UUID.randomUUID().toString, path: String = "", querystring: Option[String] = None, refererUri: Option[String] = None, diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/ApiRequestEnrichmentSpec.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/ApiRequestEnrichmentSpec.scala index 9b1efb54f..39cd3eb4e 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/ApiRequestEnrichmentSpec.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/ApiRequestEnrichmentSpec.scala @@ -12,15 +12,18 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.beam -package enrichments +package com.snowplowanalytics.snowplow.enrich.beam.enrichments import java.nio.file.Paths import cats.syntax.option._ + +import io.circe.literal._ + +import com.snowplowanalytics.snowplow.enrich.beam.{CI, Enrich, SpecHelpers} + import com.spotify.scio.io.PubsubIO import com.spotify.scio.testing._ -import io.circe.literal._ object ApiRequestEnrichmentSpec { val contexts = @@ -51,8 +54,8 @@ class ApiRequestEnrichmentSpec extends PipelineSpec { "--raw=in", "--enriched=out", "--bad=bad", - "--resolver=" + Paths.get(getClass.getResource("/iglu_resolver.json").toURI()), - "--enrichments=" + Paths.get(getClass.getResource("/api_request").toURI()) + "--resolver=" + Paths.get(getClass.getResource("/iglu_resolver.json").toURI), + "--enrichments=" + Paths.get(getClass.getResource("/api_request").toURI) ) .input(PubsubIO.readCoder[Array[Byte]]("in"), raw) .distCache(DistCacheIO(""), List.empty[Either[String, String]]) diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/SqlQueryEnrichmentSpec.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/SqlQueryEnrichmentSpec.scala index 6481ea00a..41d8d8adc 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/SqlQueryEnrichmentSpec.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/SqlQueryEnrichmentSpec.scala @@ -12,15 +12,18 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.beam -package enrichments +package com.snowplowanalytics.snowplow.enrich.beam.enrichments import java.nio.file.Paths +import io.circe.literal._ + import cats.syntax.option._ + import com.spotify.scio.io.PubsubIO import com.spotify.scio.testing._ -import io.circe.literal._ + +import com.snowplowanalytics.snowplow.enrich.beam.{CI, Enrich, SpecHelpers} object SqlQueryEnrichmentSpec { val contexts = @@ -48,8 +51,8 @@ class SqlQueryEnrichmentSpec extends PipelineSpec { "--raw=in", "--enriched=out", "--bad=bad", - "--resolver=" + Paths.get(getClass.getResource("/iglu_resolver.json").toURI()), - "--enrichments=" + Paths.get(getClass.getResource("/sql_query").toURI()) + "--resolver=" + Paths.get(getClass.getResource("/iglu_resolver.json").toURI), + "--enrichments=" + Paths.get(getClass.getResource("/sql_query").toURI) ) .input(PubsubIO.readCoder[Array[Byte]]("in"), raw) .distCache(DistCacheIO(""), List.empty[Either[String, String]]) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala index 18243ba69..6133d4095 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala @@ -10,25 +10,27 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package adapters +package com.snowplowanalytics.snowplow.enrich.common.adapters import java.time.Instant import cats.Monad import cats.data.{NonEmptyList, Validated} + import cats.effect.Clock -import cats.syntax.functor._ -import cats.syntax.validated._ +import cats.implicits._ + +import io.circe.Json + import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup import com.snowplowanalytics.iglu.client.Client + import com.snowplowanalytics.snowplow.badrows._ -import io.circe.Json -import loaders.CollectorPayload -import registry._ -import registry.snowplow._ -import utils.HttpClient +import com.snowplowanalytics.snowplow.enrich.common.adapters.registry._ +import com.snowplowanalytics.snowplow.enrich.common.adapters.registry.snowplow.{RedirectAdapter, Tp1Adapter, Tp2Adapter} +import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload +import com.snowplowanalytics.snowplow.enrich.common.utils.HttpClient /** * The AdapterRegistry lets us convert a CollectorPayload into one or more RawEvents, using a given @@ -95,14 +97,15 @@ class AdapterRegistry(remoteAdapters: Map[(String, String), RemoteAdapter] = Map processor: Processor ): F[Validated[BadRow, NonEmptyList[RawEvent]]] = (adapters.get((payload.api.vendor, payload.api.version)) match { - case Some(adapter) => adapter.toRawEvents(payload, client) + case Some(adapter) => + adapter.toRawEvents(payload, client) case _ => - val f = FailureDetails.AdapterFailure.InputData( + val f: FailureDetails.AdapterFailureOrTrackerProtocolViolation = FailureDetails.AdapterFailure.InputData( "vendor/version", Some(s"${payload.api.vendor}/${payload.api.version}"), "vendor/version combination is not supported" ) - Monad[F].pure(f.invalidNel) + Monad[F].pure(f.invalidNel[NonEmptyList[RawEvent]]) }).map(_.leftMap(enrichFailure(_, payload, payload.api.vendor, payload.api.version, processor))) private def enrichFailure( diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala index 83b0c3728..83af6f59c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala @@ -10,10 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package adapters -package registry -package snowplow +package com.snowplowanalytics.snowplow.enrich.common.adapters.registry.snowplow import cats.Monad import cats.data.{NonEmptyList, ValidatedNel} @@ -26,15 +23,18 @@ import cats.effect.Clock import io.circe._ import io.circe.syntax._ -import com.snowplowanalytics.iglu.client.Client -import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} import com.snowplowanalytics.iglu.core.circe.CirceIgluCodecs._ +import com.snowplowanalytics.iglu.client.Client +import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup + import com.snowplowanalytics.snowplow.badrows.FailureDetails -import loaders.CollectorPayload -import utils.{HttpClient, ConversionUtils => CU, JsonUtils => JU} +import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent +import com.snowplowanalytics.snowplow.enrich.common.adapters.registry.Adapter +import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload +import com.snowplowanalytics.snowplow.enrich.common.utils.{HttpClient, JsonUtils => JU, ConversionUtils => CU} /** * The Redirect Adapter is essentially a pre-processor for diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala index 6b1bdcd6c..0ee83458c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala @@ -10,22 +10,25 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package adapters -package registry -package snowplow +package com.snowplowanalytics.snowplow.enrich.common.adapters.registry.snowplow import cats.Monad import cats.data.{NonEmptyList, ValidatedNel} + import cats.effect.Clock import cats.syntax.validated._ + +import io.circe.Json + import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup -import com.snowplowanalytics.snowplow.badrows._ -import io.circe.Json -import loaders.CollectorPayload -import utils.HttpClient +import com.snowplowanalytics.snowplow.badrows.FailureDetails + +import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent +import com.snowplowanalytics.snowplow.enrich.common.adapters.registry.Adapter +import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload +import com.snowplowanalytics.snowplow.enrich.common.utils.HttpClient /** Version 1 of the Tracker Protocol is GET only. All data comes in on the querystring. */ object Tp1Adapter extends Adapter { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala index 14fe4dad4..10051a596 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala @@ -10,17 +10,17 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package adapters -package registry -package snowplow +package com.snowplowanalytics.snowplow.enrich.common.adapters.registry.snowplow import cats.Monad import cats.data.{EitherT, NonEmptyList, Validated, ValidatedNel} import cats.data.Validated._ import cats.implicits._ + import cats.effect.Clock +import io.circe.Json + import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup import com.snowplowanalytics.iglu.core.{SchemaCriterion, SelfDescribingData} @@ -28,10 +28,11 @@ import com.snowplowanalytics.iglu.core.circe.instances._ import com.snowplowanalytics.snowplow.badrows.FailureDetails -import io.circe.Json - -import loaders.CollectorPayload -import utils.{HttpClient, JsonUtils => JU} +import com.snowplowanalytics.snowplow.enrich.common.RawEventParameters +import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent +import com.snowplowanalytics.snowplow.enrich.common.adapters.registry.Adapter +import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload +import com.snowplowanalytics.snowplow.enrich.common.utils.{HttpClient, JsonUtils => JU} /** * Version 2 of the Tracker Protocol supports GET and POST. Note that with POST, data can still be diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala index 699d21691..015e3f5c8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala @@ -10,32 +10,35 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments +package com.snowplowanalytics.snowplow.enrich.common.enrichments import cats.Monad import cats.data.{EitherT, NonEmptyList, ValidatedNel} + import cats.effect.Clock import cats.implicits._ import io.circe._ import io.circe.syntax._ -import com.snowplowanalytics.iglu.client.Client -import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} import com.snowplowanalytics.iglu.core.circe.instances._ +import com.snowplowanalytics.iglu.client.Client +import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup + import com.snowplowanalytics.forex.CreateForex import com.snowplowanalytics.maxmind.iplookups.CreateIpLookups import com.snowplowanalytics.refererparser.CreateParser import com.snowplowanalytics.weather.providers.openweather.CreateOWM -import registry._ -import registry.apirequest.ApiRequestEnrichment -import registry.pii.PiiPseudonymizerEnrichment -import registry.sqlquery.SqlQueryEnrichment -import utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf._ + +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry._ +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest.ApiRequestEnrichment +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.pii.PiiPseudonymizerEnrichment +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery.SqlQueryEnrichment /** Companion which holds a constructor for the EnrichmentRegistry. */ object EnrichmentRegistry { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala index 49c1087ff..5dfe3dc95 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala @@ -10,20 +10,22 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry -import cats.data.ValidatedNel -import cats.data.Validated +import java.net.{Inet4Address, Inet6Address} + +import scala.util.Try + +import cats.data.{Validated, ValidatedNel} import cats.syntax.either._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} -import io.circe._ -import utils.CirceUtils +import io.circe.Json -import java.net.{Inet4Address, Inet6Address} import com.google.common.net.{InetAddresses => GuavaInetAddress} -import scala.util.Try +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.AnonIpConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object. Lets us create a AnonIpConf from a Json. */ object AnonIpEnrichment extends ParseableEnrichment { @@ -32,7 +34,7 @@ object AnonIpEnrichment extends ParseableEnrichment { /** * Creates an AnonIpEnrichment instance from a Json. - * @param c The anon_ip enrichment JSON + * @param config The anon_ip enrichment JSON * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return an AnonIpEnrichment configuration */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala index 0c1653633..f4f1cd874 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala @@ -10,16 +10,19 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import cats.data.{NonEmptyList, ValidatedNel} import cats.implicits._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + import io.circe._ -import utils.MapTransformer.SourceMap -import utils.CirceUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + +import com.snowplowanalytics.snowplow.enrich.common.QueryStringParameters +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.CampaignAttributionConf +import com.snowplowanalytics.snowplow.enrich.common.utils.MapTransformer.SourceMap +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object. Lets us create a CampaignAttributionEnrichment from a Json */ object CampaignAttributionEnrichment extends ParseableEnrichment { @@ -97,7 +100,7 @@ final case class MarketingCampaign( * @param termParameters List of marketing term parameters * @param contentParameters List of marketing content parameters * @param campaignParameters List of marketing campaign parameters - * @param mktClick Map of click ID parameters to networks + * @param clickIdParameters Map of click ID parameters to networks */ final case class CampaignAttributionEnrichment( mediumParameters: List[String], @@ -116,7 +119,7 @@ final case class CampaignAttributionEnrichment( * @return Option boxing the value of the campaign parameter */ private def getFirstParameter(parameterList: List[String], sourceMap: SourceMap): Option[String] = - parameterList.find(sourceMap.contains(_)).map(sourceMap(_)) + parameterList.find(sourceMap.contains).map(sourceMap(_)) /** * Extract the marketing fields from a URL. diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala index 37697ff5f..13721cebb 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala @@ -10,20 +10,19 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import cats.data.ValidatedNel import cats.syntax.either._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} - import io.circe._ import io.circe.syntax._ import org.apache.http.message.BasicHeaderValueParser +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} -import utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.CookieExtractorConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils object CookieExtractorEnrichment extends ParseableEnrichment { override val supportedSchema = diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala index ddd65818d..ad0d81492 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala @@ -10,23 +10,28 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.time.ZonedDateTime import cats.Monad import cats.data.{EitherT, NonEmptyList, ValidatedNel} import cats.implicits._ + +import io.circe._ + import com.snowplowanalytics.forex.{CreateForex, Forex} import com.snowplowanalytics.forex.model._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} -import com.snowplowanalytics.snowplow.badrows._ -import io.circe._ + import org.joda.money.CurrencyUnit import org.joda.time.DateTime -import utils.CirceUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + +import com.snowplowanalytics.snowplow.badrows.FailureDetails + +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.CurrencyConversionConf /** Companion object. Lets us create an CurrencyConversionEnrichment instance from a Json. */ object CurrencyConversionEnrichment extends ParseableEnrichment { @@ -101,10 +106,10 @@ final case class CurrencyConversionEnrichment[F[_]: Monad]( /** * Attempt to convert if the initial currency and value are both defined - * @param inputCurrency Option boxing the initial currency if it is present + * @param initialCurrency Option boxing the initial currency if it is present * @param value Option boxing the amount to convert * @return None.success if the inputs were not both defined, - * otherwise Validation[Option[_]] boxing the result of the conversion + * otherwise `Validation[Option[_]]` boxing the result of the conversion */ private def performConversion( initialCurrency: Option[Either[FailureDetails.EnrichmentFailure, CurrencyUnit]], diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala new file mode 100644 index 000000000..e4f6dab0d --- /dev/null +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry + +import java.net.URI + +import cats.{Functor, Monad} +import cats.data.EitherT + +import org.joda.money.CurrencyUnit + +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} + +import com.snowplowanalytics.forex.CreateForex +import com.snowplowanalytics.forex.model.AccountType +import com.snowplowanalytics.maxmind.iplookups.CreateIpLookups +import com.snowplowanalytics.refererparser.CreateParser +import com.snowplowanalytics.weather.providers.openweather.CreateOWM + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest.{ + ApiRequestEnrichment, + CreateApiRequestEnrichment, + HttpApi +} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery.{CreateSqlQueryEnrichment, Rdbms, SqlQueryEnrichment} + +sealed trait EnrichmentConf { + def schemaKey: SchemaKey = + SchemaKey( + "com.acme", + "placeholder", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + + /** + * List of files, such as local DBs that need to be downloaded and distributed across workers + * First element of pair is URI to download file from, second is a local path to store it in + */ + def filesToCache: List[(URI, String)] = Nil +} + +object EnrichmentConf { + + final case class ApiRequestConf( + override val schemaKey: SchemaKey, + inputs: List[apirequest.Input], + api: HttpApi, + outputs: List[apirequest.Output], + cache: apirequest.Cache + ) extends EnrichmentConf { + def enrichment[F[_]: CreateApiRequestEnrichment]: F[ApiRequestEnrichment[F]] = + ApiRequestEnrichment[F](this) + } + + final case class PiiPseudonymizerConf( + fieldList: List[pii.PiiField], + emitIdentificationEvent: Boolean, + strategy: pii.PiiStrategy + ) extends EnrichmentConf { + def enrichment: pii.PiiPseudonymizerEnrichment = + pii.PiiPseudonymizerEnrichment(fieldList, emitIdentificationEvent, strategy) + } + + final case class SqlQueryConf( + override val schemaKey: SchemaKey, + inputs: List[sqlquery.Input], + db: Rdbms, + query: SqlQueryEnrichment.Query, + output: sqlquery.Output, + cache: SqlQueryEnrichment.Cache + ) extends EnrichmentConf { + def enrichment[F[_]: Monad: CreateSqlQueryEnrichment]: F[SqlQueryEnrichment[F]] = + SqlQueryEnrichment[F](this) + } + + final case class AnonIpConf(octets: AnonIPv4Octets.AnonIPv4Octets, segments: AnonIPv6Segments.AnonIPv6Segments) extends EnrichmentConf { + override val filesToCache: List[(URI, String)] = Nil + def enrichment: AnonIpEnrichment = AnonIpEnrichment(octets, segments) + } + + final case class CampaignAttributionConf( + mediumParameters: List[String], + sourceParameters: List[String], + termParameters: List[String], + contentParameters: List[String], + campaignParameters: List[String], + clickIdParameters: List[(String, String)] + ) extends EnrichmentConf { + def enrichment: CampaignAttributionEnrichment = + CampaignAttributionEnrichment( + mediumParameters, + sourceParameters, + termParameters, + contentParameters, + campaignParameters, + clickIdParameters + ) + } + + final case class CookieExtractorConf(cookieNames: List[String]) extends EnrichmentConf { + def enrichment: CookieExtractorEnrichment = CookieExtractorEnrichment(cookieNames) + } + + final case class CurrencyConversionConf( + override val schemaKey: SchemaKey, + accountType: AccountType, + apiKey: String, + baseCurrency: CurrencyUnit + ) extends EnrichmentConf { + def enrichment[F[_]: Monad: CreateForex]: F[CurrencyConversionEnrichment[F]] = + CurrencyConversionEnrichment[F](this) + } + + final case class EventFingerprintConf(algorithm: String => String, excludedParameters: List[String]) extends EnrichmentConf { + def enrichment: EventFingerprintEnrichment = + EventFingerprintEnrichment(algorithm, excludedParameters) + } + + final case class HttpHeaderExtractorConf(headersPattern: String) extends EnrichmentConf { + def enrichment: HttpHeaderExtractorEnrichment = HttpHeaderExtractorEnrichment(headersPattern) + } + + final case class IabConf( + override val schemaKey: SchemaKey, + ipFile: (URI, String), + excludeUaFile: (URI, String), + includeUaFile: (URI, String) + ) extends EnrichmentConf { + override val filesToCache: List[(URI, String)] = List(ipFile, excludeUaFile, includeUaFile) + def enrichment[F[_]: Monad: CreateIabClient]: F[IabEnrichment] = + IabEnrichment[F](this) + } + + final case class IpLookupsConf( + geoFile: Option[(URI, String)], + ispFile: Option[(URI, String)], + domainFile: Option[(URI, String)], + connectionTypeFile: Option[(URI, String)] + ) extends EnrichmentConf { + override val filesToCache: List[(URI, String)] = + List(geoFile, ispFile, domainFile, connectionTypeFile).flatten + def enrichment[F[_]: Functor: CreateIpLookups]: F[IpLookupsEnrichment[F]] = + IpLookupsEnrichment[F](this) + } + + final case class JavascriptScriptConf(override val schemaKey: SchemaKey, rawFunction: String) extends EnrichmentConf { + def enrichment: JavascriptScriptEnrichment = JavascriptScriptEnrichment(schemaKey, rawFunction) + } + + final case class RefererParserConf(refererDatabase: (URI, String), internalDomains: List[String]) extends EnrichmentConf { + override val filesToCache: List[(URI, String)] = List(refererDatabase) + def enrichment[F[_]: Monad: CreateParser]: EitherT[F, String, RefererParserEnrichment] = + RefererParserEnrichment[F](this) + } + + final case class UaParserConf(override val schemaKey: SchemaKey, uaDatabase: Option[(URI, String)]) extends EnrichmentConf { + override val filesToCache: List[(URI, String)] = List(uaDatabase).flatten + def enrichment[F[_]: Monad: CreateUaParser]: EitherT[F, String, UaParserEnrichment] = + UaParserEnrichment[F](this) + } + + final case class UserAgentUtilsConf(override val schemaKey: SchemaKey) extends EnrichmentConf { + def enrichment: UserAgentUtilsEnrichment = UserAgentUtilsEnrichment(schemaKey) + } + + final case class WeatherConf( + override val schemaKey: SchemaKey, + apiHost: String, + apiKey: String, + timeout: Int, + cacheSize: Int, + geoPrecision: Int + ) extends EnrichmentConf { + def enrichment[F[_]: Monad: CreateOWM]: EitherT[F, String, WeatherEnrichment[F]] = + WeatherEnrichment[F](this) + } + + final case class YauaaConf(cacheSize: Option[Int]) extends EnrichmentConf { + def enrichment: YauaaEnrichment = YauaaEnrichment(cacheSize) + } +} diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala index 4e9c9bd7e..607012426 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala @@ -10,16 +10,19 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import cats.data.{NonEmptyList, ValidatedNel} import cats.implicits._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + import io.circe._ + import org.apache.commons.codec.digest.DigestUtils -import utils.CirceUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.EventFingerprintConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Lets us create an EventFingerprintEnrichment from a Json. */ object EventFingerprintEnrichment extends ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala index cd7f949c0..64d934ebd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala @@ -10,18 +10,18 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import cats.data.ValidatedNel import cats.syntax.either._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} - import io.circe._ import io.circe.syntax._ -import utils.CirceUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.HttpHeaderExtractorConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils object HttpHeaderExtractorEnrichment extends ParseableEnrichment { override val supportedSchema = diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala index c9858ea83..79500cf1e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala @@ -10,31 +10,31 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.io.File import java.net.URI -import inet.ipaddr.HostName - import cats.{Id, Monad} import cats.data.{NonEmptyList, ValidatedNel} import cats.effect.Sync import cats.implicits._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} +import org.joda.time.DateTime -import com.snowplowanalytics.iab.spidersandrobotsclient.IabClient -import com.snowplowanalytics.snowplow.badrows.FailureDetails +import inet.ipaddr.HostName import io.circe._ import io.circe.generic.auto._ import io.circe.syntax._ -import org.joda.time.DateTime +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} + +import com.snowplowanalytics.iab.spidersandrobotsclient.IabClient +import com.snowplowanalytics.snowplow.badrows.FailureDetails -import utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.IabConf /** Companion object. Lets us create an IabEnrichment instance from a Json. */ object IabEnrichment extends ParseableEnrichment { @@ -111,10 +111,8 @@ object IabEnrichment extends ParseableEnrichment { /** * Contains enrichments based on IAB Spiders&Robots lookup. - * @param ipFile (Full URI to the IAB excluded IP list, database name) - * @param excludeUaFile (Full URI to the IAB excluded user agent list, database name) - * @param includeUaFile (Full URI to the IAB included user agent list, database name) - * @param localMode Whether to use the local database file. Enabled for tests. + * @param schemaKey enrichment's static Iglu Schema Key + * @param iabClient worker object */ final case class IabEnrichment(schemaKey: SchemaKey, iabClient: IabClient) extends Enrichment { val outputSchema = diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala index 3fb81b183..956221f55 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala @@ -10,26 +10,25 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments -package registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.net.URI -import inet.ipaddr.HostName - import cats.Functor import cats.data.{NonEmptyList, ValidatedNel} import cats.implicits._ -import com.snowplowanalytics.maxmind.iplookups._ -import com.snowplowanalytics.maxmind.iplookups.model._ +import io.circe._ + +import inet.ipaddr.HostName import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} -import io.circe._ +import com.snowplowanalytics.maxmind.iplookups._ +import com.snowplowanalytics.maxmind.iplookups.model._ -import utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.IpLookupsConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object. Lets us create an IpLookupsEnrichment instance from a Json. */ object IpLookupsEnrichment extends ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala index 7ee71405c..630a6e901 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala @@ -10,24 +10,24 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import cats.data.{NonEmptyList, ValidatedNel} import cats.implicits._ +import io.circe._ +import io.circe.parser._ + +import javax.script._ + import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} import com.snowplowanalytics.iglu.core.circe.implicits._ import com.snowplowanalytics.snowplow.badrows.FailureDetails -import javax.script._ - -import io.circe._ -import io.circe.parser._ - -import outputs.EnrichedEvent -import utils.{CirceUtils, ConversionUtils} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.JavascriptScriptConf +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import com.snowplowanalytics.snowplow.enrich.common.utils.{CirceUtils, ConversionUtils} object JavascriptScriptEnrichment extends ParseableEnrichment { override val supportedSchema = diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala index c47603ad6..5bc29aedd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala @@ -10,20 +10,22 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.net.URI import cats.Monad import cats.data.{EitherT, NonEmptyList, ValidatedNel} import cats.implicits._ + +import io.circe.Json + import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + import com.snowplowanalytics.refererparser._ -import io.circe.Json -import utils.{ConversionUtils => CU} -import utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.RefererParserConf +import com.snowplowanalytics.snowplow.enrich.common.utils.{ConversionUtils => CU, CirceUtils} /** Companion object. Lets us create a RefererParserEnrichment from a Json */ object RefererParserEnrichment extends ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala index cb9931a8b..03dd5dabc 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala @@ -9,28 +9,29 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.io.{FileInputStream, InputStream} import java.net.URI import cats.{Id, Monad} import cats.data.{EitherT, NonEmptyList, ValidatedNel} + import cats.effect.Sync import cats.implicits._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} - -import com.snowplowanalytics.snowplow.badrows.FailureDetails - -import io.circe._ +import io.circe.Json import io.circe.syntax._ import ua_parser.Parser import ua_parser.Client -import utils.CirceUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} + +import com.snowplowanalytics.snowplow.badrows.FailureDetails + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.UaParserConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object. Lets us create a UaParserEnrichment from a Json. */ object UaParserEnrichment extends ParseableEnrichment { @@ -105,7 +106,7 @@ final case class UaParserEnrichment(schemaKey: SchemaKey, parser: Parser) extend /** * Extracts the client attributes from a useragent string, using UserAgentEnrichment. * @param useragent to extract from. Should be encoded, i.e. not previously decoded. - * @return the json or the message of the exception, boxed in a Scalaz Validation + * @return the json or the message of the bad row details */ def extractUserAgent(useragent: String): Either[FailureDetails.EnrichmentFailure, SelfDescribingData[Json]] = Either diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala index ef11dbf33..78725ea8e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala @@ -9,18 +9,24 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import scala.util.control.NonFatal import cats.data.ValidatedNel import cats.syntax.either._ import cats.syntax.option._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} -import com.snowplowanalytics.snowplow.badrows._ -import eu.bitwalker.useragentutils._ + import io.circe._ + +import eu.bitwalker.useragentutils._ + +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + +import com.snowplowanalytics.snowplow.badrows.FailureDetails + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.UserAgentUtilsConf + import org.slf4j.LoggerFactory object UserAgentUtilsEnrichmentConfig extends ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala index 804156d75..2fa748057 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala @@ -10,8 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.lang.{Float => JFloat} import java.time.{Instant, ZoneOffset, ZonedDateTime} @@ -23,19 +22,21 @@ import cats.Monad import cats.data.{EitherT, NonEmptyList, ValidatedNel} import cats.implicits._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} -import com.snowplowanalytics.snowplow.badrows.FailureDetails - -import com.snowplowanalytics.weather.providers.openweather._ -import com.snowplowanalytics.weather.providers.openweather.responses._ +import org.joda.time.{DateTime, DateTimeZone} import io.circe._ import io.circe.generic.auto._ import io.circe.syntax._ -import org.joda.time.{DateTime, DateTimeZone} +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} + +import com.snowplowanalytics.snowplow.badrows.FailureDetails + +import com.snowplowanalytics.weather.providers.openweather._ +import com.snowplowanalytics.weather.providers.openweather.responses._ -import utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.WeatherConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object. Lets us create an WeatherEnrichment instance from a Json */ object WeatherEnrichment extends ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala index 0cceb153a..9f0b9de36 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala @@ -10,8 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import scala.collection.JavaConverters._ @@ -21,11 +20,12 @@ import cats.syntax.either._ import io.circe.Json import io.circe.syntax._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} - import nl.basjes.parse.useragent.{UserAgent, UserAgentAnalyzer} -import utils.CirceUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.YauaaConf +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object to create an instance of YauaaEnrichment from the configuration. */ object YauaaEnrichment extends ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala index 65f24b2cd..fd589350a 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala @@ -10,10 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments -package registry -package apirequest +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest import java.util.UUID @@ -21,17 +18,19 @@ import cats.{Id, Monad} import cats.data.{EitherT, NonEmptyList, ValidatedNel} import cats.implicits._ +import io.circe._ +import io.circe.generic.auto._ + import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} import com.snowplowanalytics.iglu.core.circe.implicits._ -import com.snowplowanalytics.lrumap._ +import com.snowplowanalytics.lrumap._ import com.snowplowanalytics.snowplow.badrows.FailureDetails -import io.circe._ -import io.circe.generic.auto._ - -import outputs.EnrichedEvent -import utils.{CirceUtils, HttpClient} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.{Enrichment, ParseableEnrichment} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.ApiRequestConf +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import com.snowplowanalytics.snowplow.enrich.common.utils.{CirceUtils, HttpClient} object ApiRequestEnrichment extends ParseableEnrichment { override val supportedSchema = diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala index 6ed023b1f..70e996223 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala @@ -10,162 +10,22 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.net.URI -import cats.{Functor, Monad} -import cats.data.{EitherT, ValidatedNel} +import cats.data.ValidatedNel import cats.syntax.either._ -import com.snowplowanalytics.forex.CreateForex -import com.snowplowanalytics.forex.model.AccountType -import com.snowplowanalytics.iglu.core._ -import com.snowplowanalytics.maxmind.iplookups.CreateIpLookups -import com.snowplowanalytics.refererparser.CreateParser -import com.snowplowanalytics.weather.providers.openweather.CreateOWM + import io.circe._ -import org.joda.money.CurrencyUnit -import apirequest._ -import sqlquery._ -import utils.ConversionUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey} + +import com.snowplowanalytics.snowplow.enrich.common.utils.ConversionUtils /** Trait inherited by every enrichment config case class */ trait Enrichment -sealed trait EnrichmentConf { - def schemaKey: SchemaKey = - SchemaKey( - "com.acme", - "placeholder", - "jsonschema", - SchemaVer.Full(1, 0, 0) - ) - def filesToCache: List[(URI, String)] = Nil -} -final case class ApiRequestConf( - override val schemaKey: SchemaKey, - inputs: List[apirequest.Input], - api: HttpApi, - outputs: List[apirequest.Output], - cache: apirequest.Cache -) extends EnrichmentConf { - def enrichment[F[_]: CreateApiRequestEnrichment]: F[ApiRequestEnrichment[F]] = - ApiRequestEnrichment[F](this) -} -final case class PiiPseudonymizerConf( - fieldList: List[pii.PiiField], - emitIdentificationEvent: Boolean, - strategy: pii.PiiStrategy -) extends EnrichmentConf { - def enrichment: pii.PiiPseudonymizerEnrichment = - pii.PiiPseudonymizerEnrichment(fieldList, emitIdentificationEvent, strategy) -} -final case class SqlQueryConf( - override val schemaKey: SchemaKey, - inputs: List[sqlquery.Input], - db: Rdbms, - query: SqlQueryEnrichment.Query, - output: sqlquery.Output, - cache: SqlQueryEnrichment.Cache -) extends EnrichmentConf { - def enrichment[F[_]: Monad: CreateSqlQueryEnrichment]: F[SqlQueryEnrichment[F]] = - SqlQueryEnrichment[F](this) -} -final case class AnonIpConf(octets: AnonIPv4Octets.AnonIPv4Octets, segments: AnonIPv6Segments.AnonIPv6Segments) extends EnrichmentConf { - override val filesToCache: List[(URI, String)] = Nil - def enrichment: AnonIpEnrichment = AnonIpEnrichment(octets, segments) -} -final case class CampaignAttributionConf( - mediumParameters: List[String], - sourceParameters: List[String], - termParameters: List[String], - contentParameters: List[String], - campaignParameters: List[String], - clickIdParameters: List[(String, String)] -) extends EnrichmentConf { - def enrichment: CampaignAttributionEnrichment = - CampaignAttributionEnrichment( - mediumParameters, - sourceParameters, - termParameters, - contentParameters, - campaignParameters, - clickIdParameters - ) -} -final case class CookieExtractorConf(cookieNames: List[String]) extends EnrichmentConf { - def enrichment: CookieExtractorEnrichment = CookieExtractorEnrichment(cookieNames) -} -final case class CurrencyConversionConf( - override val schemaKey: SchemaKey, - accountType: AccountType, - apiKey: String, - baseCurrency: CurrencyUnit -) extends EnrichmentConf { - def enrichment[F[_]: Monad: CreateForex]: F[CurrencyConversionEnrichment[F]] = - CurrencyConversionEnrichment[F](this) -} -final case class EventFingerprintConf(algorithm: String => String, excludedParameters: List[String]) extends EnrichmentConf { - def enrichment: EventFingerprintEnrichment = - EventFingerprintEnrichment(algorithm, excludedParameters) -} -final case class HttpHeaderExtractorConf(headersPattern: String) extends EnrichmentConf { - def enrichment: HttpHeaderExtractorEnrichment = HttpHeaderExtractorEnrichment(headersPattern) -} -final case class IabConf( - override val schemaKey: SchemaKey, - ipFile: (URI, String), - excludeUaFile: (URI, String), - includeUaFile: (URI, String) -) extends EnrichmentConf { - override val filesToCache: List[(URI, String)] = List(ipFile, excludeUaFile, includeUaFile) - def enrichment[F[_]: Monad: CreateIabClient]: F[IabEnrichment] = - IabEnrichment[F](this) -} -final case class IpLookupsConf( - geoFile: Option[(URI, String)], - ispFile: Option[(URI, String)], - domainFile: Option[(URI, String)], - connectionTypeFile: Option[(URI, String)] -) extends EnrichmentConf { - override val filesToCache: List[(URI, String)] = - List(geoFile, ispFile, domainFile, connectionTypeFile).flatten - def enrichment[F[_]: Functor: CreateIpLookups]: F[IpLookupsEnrichment[F]] = - IpLookupsEnrichment[F](this) -} -final case class JavascriptScriptConf(override val schemaKey: SchemaKey, rawFunction: String) extends EnrichmentConf { - def enrichment: JavascriptScriptEnrichment = JavascriptScriptEnrichment(schemaKey, rawFunction) -} -final case class RefererParserConf(refererDatabase: (URI, String), internalDomains: List[String]) extends EnrichmentConf { - override val filesToCache: List[(URI, String)] = List(refererDatabase) - def enrichment[F[_]: Monad: CreateParser]: EitherT[F, String, RefererParserEnrichment] = - RefererParserEnrichment[F](this) -} -final case class UaParserConf(override val schemaKey: SchemaKey, uaDatabase: Option[(URI, String)]) extends EnrichmentConf { - override val filesToCache: List[(URI, String)] = List(uaDatabase).flatten - def enrichment[F[_]: Monad: CreateUaParser]: EitherT[F, String, UaParserEnrichment] = - UaParserEnrichment[F](this) -} -final case class UserAgentUtilsConf(override val schemaKey: SchemaKey) extends EnrichmentConf { - def enrichment: UserAgentUtilsEnrichment = UserAgentUtilsEnrichment(schemaKey) -} -final case class WeatherConf( - override val schemaKey: SchemaKey, - apiHost: String, - apiKey: String, - timeout: Int, - cacheSize: Int, - geoPrecision: Int -) extends EnrichmentConf { - def enrichment[F[_]: Monad: CreateOWM]: EitherT[F, String, WeatherEnrichment[F]] = - WeatherEnrichment[F](this) -} -final case class YauaaConf(cacheSize: Option[Int]) extends EnrichmentConf { - def enrichment: YauaaEnrichment = YauaaEnrichment(cacheSize) -} - /** Trait to hold helpers relating to enrichment config */ trait ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala index d13733105..474922111 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala @@ -10,9 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry -package pii +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.pii import scala.collection.JavaConverters._ import scala.collection.mutable.MutableList @@ -20,24 +18,26 @@ import scala.collection.mutable.MutableList import cats.data.ValidatedNel import cats.implicits._ +import io.circe._ +import io.circe.jackson._ +import io.circe.syntax._ + import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode, TextNode} import com.jayway.jsonpath.{Configuration, JsonPath => JJsonPath} import com.jayway.jsonpath.MapFunction -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} - -import io.circe._ -import io.circe.jackson._ -import io.circe.syntax._ - import org.apache.commons.codec.digest.DigestUtils -import adapters.registry.Adapter -import outputs.EnrichedEvent -import serializers._ -import utils.CirceUtils +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} + +import com.snowplowanalytics.snowplow.enrich.common.adapters.registry.Adapter +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.PiiPseudonymizerConf +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.{Enrichment, ParseableEnrichment} +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.pii.serializers._ +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils /** Companion object. Lets us create a PiiPseudonymizerEnrichment from a Json. */ object PiiPseudonymizerEnrichment extends ParseableEnrichment { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/CreateSqlQueryEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/CreateSqlQueryEnrichment.scala index 3ca02cf07..92112d7e8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/CreateSqlQueryEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/CreateSqlQueryEnrichment.scala @@ -13,11 +13,12 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery import cats.Id + import cats.effect.Sync import cats.syntax.functor._ import cats.syntax.flatMap._ -import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.SqlQueryConf +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.SqlQueryConf /** Initialize resources, necessary for SQL Query enrichment: cache and connection */ sealed trait CreateSqlQueryEnrichment[F[_]] { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala index b669cd351..c8fb537ac 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala @@ -10,10 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments -package registry -package sqlquery +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery import scala.collection.immutable.IntMap @@ -21,15 +18,17 @@ import cats.Monad import cats.data.{EitherT, NonEmptyList, ValidatedNel} import cats.implicits._ +import io.circe._ +import io.circe.generic.semiauto._ + import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SelfDescribingData} import com.snowplowanalytics.snowplow.badrows.FailureDetails -import io.circe._ -import io.circe.generic.semiauto._ - -import outputs.EnrichedEvent -import utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.SqlQueryConf +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.{Enrichment, ParseableEnrichment} /** Lets us create an SqlQueryConf from a Json */ object SqlQueryEnrichment extends ParseableEnrichment { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala index f3cafa750..ebc89d353 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala @@ -10,8 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import cats.Id import cats.data.{NonEmptyList, Validated, ValidatedNel} @@ -19,11 +18,15 @@ import cats.implicits._ import com.snowplowanalytics.forex.CreateForex._ import com.snowplowanalytics.forex.model._ + import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} -import com.snowplowanalytics.snowplow.badrows._ + +import com.snowplowanalytics.snowplow.badrows.FailureDetails +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.CurrencyConversionConf import org.joda.money.CurrencyUnit import org.joda.time.DateTime + import org.specs2.Specification import org.specs2.matcher.DataTables @@ -42,11 +45,10 @@ class CurrencyConversionEnrichmentSpec extends Specification with DataTables { """ lazy val validAppKey = sys.env - .get(OerApiKey) - .getOrElse( - throw new IllegalStateException( - s"No ${OerApiKey} environment variable found, test should have been skipped" - ) + .getOrElse(OerApiKey, + throw new IllegalStateException( + s"No $OerApiKey environment variable found, test should have been skipped" + ) ) type Result = ValidatedNel[ FailureDetails.EnrichmentFailure, diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala index 0282a4063..49460c360 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala @@ -14,12 +14,17 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.net.URI -import com.snowplowanalytics.forex.model._ -import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer} import io.circe.literal._ import io.circe.parser._ + import org.apache.commons.codec.binary.Base64 import org.joda.money.CurrencyUnit + +import com.snowplowanalytics.forex.model._ +import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer} + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf._ + import org.specs2.matcher.{DataTables, ValidatedMatchers} import org.specs2.mutable.Specification diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala index 5fdf789d8..d51dc85ca 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala @@ -21,6 +21,8 @@ import io.circe.literal._ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.UaParserConf + import org.specs2.matcher.DataTables import org.specs2.mutable.Specification diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala index 149fc087b..a67398c56 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala @@ -24,6 +24,8 @@ import org.joda.time.DateTime import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.WeatherConf + import org.specs2.Specification object WeatherEnrichmentSpec { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala index 11275d358..50089a985 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala @@ -10,17 +10,17 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments -package registry - -import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import io.circe.parser._ import io.circe.literal._ import nl.basjes.parse.useragent.UserAgent +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.YauaaConf + import org.specs2.matcher.ValidatedMatchers import org.specs2.mutable.Specification diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala index 703f8db17..eaf36ba21 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala @@ -10,9 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry -package apirequest +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest import cats.Id import cats.syntax.either._ @@ -21,16 +19,17 @@ import io.circe.Json import io.circe.literal._ import io.circe.parser._ -import org.specs2.Specification -import org.specs2.matcher.ValidatedMatchers -import org.specs2.mock.Mockito - import scalaj.http.HttpRequest import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} -import outputs.EnrichedEvent -import utils.HttpClient +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.ApiRequestConf +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import com.snowplowanalytics.snowplow.enrich.common.utils.HttpClient + +import org.specs2.Specification +import org.specs2.matcher.ValidatedMatchers +import org.specs2.mock.Mockito class ApiRequestEnrichmentSpec extends Specification with ValidatedMatchers with Mockito { def is = s2""" diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala index 93568490e..9f233f676 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala @@ -10,12 +10,15 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry -package apirequest +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest import cats.Id + import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} + +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.ApiRequestConf +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent + import org.specs2.Specification import org.specs2.matcher.ValidatedMatchers import org.specs2.mock.Mockito @@ -49,7 +52,6 @@ class HttpApiSpec extends Specification with ValidatedMatchers with Mockito { request must beSome("http://thishostdoesntexist31337:8123/admin/foo/November+2015/admin") } - // This one uses real actor system def e3 = { val schemaKey = SchemaKey("vendor", "name", "format", SchemaVer.Full(1, 0, 0)) val enrichment = ApiRequestConf( @@ -60,7 +62,7 @@ class HttpApiSpec extends Specification with ValidatedMatchers with Mockito { Cache(1, 1) ).enrichment[Id] - val event = new outputs.EnrichedEvent + val event = new EnrichedEvent val request = enrichment.lookup(event, Nil, Nil, None) request must beInvalid } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala index 887da6e64..96e8758a4 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala @@ -10,9 +10,7 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common -package enrichments.registry -package apirequest +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest import cats.Id import cats.data.ValidatedNel @@ -22,9 +20,11 @@ import io.circe.literal._ import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer, SelfDescribingData} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.ApiRequestConf +import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent + import org.specs2.Specification import org.specs2.matcher.ValidatedMatchers -import outputs.EnrichedEvent class InputSpec extends Specification with ValidatedMatchers { def is = s2""" @@ -253,7 +253,7 @@ class InputSpec extends Specification with ValidatedMatchers { List(Output("iglu:someschema", JsonOutput("$").some)), Cache(10, 5) ).enrichment[Id] - val event = new outputs.EnrichedEvent + val event = new EnrichedEvent event.setUser_id("chuwy") // time in true_tstamp won't be found val request = enrichment.lookup(event, Nil, Nil, None) @@ -276,7 +276,7 @@ class InputSpec extends Specification with ValidatedMatchers { json"""{ "somekey": "somevalue" }""" ) - input.pull(new outputs.EnrichedEvent, Nil, List(obj), None) must beValid.like { + input.pull(new EnrichedEvent, Nil, List(obj), None) must beValid.like { case Some(context) => context must beEqualTo(Map("permissive" -> "somevalue")) case None => diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala index ebaa0c714..dc3365a1a 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala @@ -10,14 +10,14 @@ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. */ -package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry -package sqlquery +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery import io.circe.parser._ import io.circe.literal._ import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer} +import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.SqlQueryConf import com.snowplowanalytics.snowplow.enrich.common.utils.CirceUtils import org.specs2.Specification