From 4d0b947badeec8e1f9f98740dfa963ccb841a596 Mon Sep 17 00:00:00 2001 From: Ben Fradet Date: Wed, 13 Mar 2019 23:32:14 +0100 Subject: [PATCH] Common: replace json4s with circe (close snowplow/snowplow#3602) --- .scalafmt.conf | 2 +- common.sbt | 10 +- .../common/EtlException.scala | 130 +--- .../common/EtlPipeline.scala | 43 +- .../common/adapters/AdapterRegistry.scala | 47 +- .../common/adapters/registry/Adapter.scala | 411 +++++++------ .../adapters/registry/CallrailAdapter.scala | 15 +- .../registry/CloudfrontAccessLogAdapter.scala | 128 ++-- .../registry/GoogleAnalyticsAdapter.scala | 57 +- .../adapters/registry/HubSpotAdapter.scala | 131 ++-- .../adapters/registry/IgluAdapter.scala | 191 +++--- .../adapters/registry/MailchimpAdapter.scala | 137 ++--- .../adapters/registry/MailgunAdapter.scala | 132 ++--- .../adapters/registry/MandrillAdapter.scala | 124 ++-- .../adapters/registry/MarketoAdapter.scala | 108 ++-- .../adapters/registry/OlarkAdapter.scala | 166 +++--- .../adapters/registry/PagerdutyAdapter.scala | 137 ++--- .../adapters/registry/PingdomAdapter.scala | 131 ++-- .../adapters/registry/RemoteAdapter.scala | 45 +- .../adapters/registry/SendgridAdapter.scala | 129 ++-- .../registry/StatusGatorAdapter.scala | 72 +-- .../adapters/registry/UnbounceAdapter.scala | 101 ++-- .../registry/UrbanAirshipAdapter.scala | 102 ++-- .../adapters/registry/VeroAdapter.scala | 98 ++- .../registry/snowplow/RedirectAdapter.scala | 81 ++- .../registry/snowplow/Tp1Adapter.scala | 24 +- .../registry/snowplow/Tp2Adapter.scala | 93 ++- .../enrichments/ClientEnrichments.scala | 52 +- .../enrichments/EnrichmentManager.scala | 35 +- .../enrichments/EnrichmentRegistry.scala | 256 ++++---- .../common/enrichments/EventEnrichments.scala | 58 +- .../common/enrichments/MiscEnrichments.scala | 59 +- .../common/enrichments/SchemaEnrichment.scala | 15 +- .../registry/AnonIpEnrichment.scala | 73 +-- .../CampaignAttributionEnrichment.scala | 68 +-- .../registry/CookieExtractorEnrichment.scala | 44 +- .../CurrencyConversionEnrichment.scala | 84 +-- .../registry/EventFingerprintEnrichment.scala | 59 +- .../HttpHeaderExtractorEnrichment.scala | 45 +- .../enrichments/registry/IabEnrichment.scala | 172 +++--- .../registry/IpLookupsEnrichment.scala | 88 ++- .../registry/JavascriptScriptEnrichment.scala | 112 ++-- .../registry/RefererParserEnrichment.scala | 58 +- .../registry/UaParserEnrichment.scala | 121 ++-- .../registry/UserAgentUtilsEnrichment.scala | 46 +- .../registry/WeatherEnrichment.scala | 112 ++-- .../registry/YauaaEnrichment.scala | 18 +- .../apirequest/ApiRequestEnrichment.scala | 104 ++-- .../registry/apirequest/Cache.scala | 14 +- .../registry/apirequest/HttpApi.scala | 39 +- .../registry/apirequest/Input.scala | 97 ++- .../registry/apirequest/Output.scala | 74 +-- .../enrichments/registry/enrichments.scala | 35 +- .../enrichments/registry/pii/Mutators.scala | 6 +- .../pii/PiiPseudonymizerEnrichment.scala | 294 +++++---- .../registry/pii/Serializers.scala | 76 --- .../enrichments/registry/pii/package.scala | 50 +- .../registry/pii/serializers.scala | 58 ++ .../enrichments/registry/sqlquery/Cache.scala | 13 +- .../enrichments/registry/sqlquery/Db.scala | 32 +- .../enrichments/registry/sqlquery/Input.scala | 168 +++--- .../registry/sqlquery/Output.scala | 220 ++++--- .../enrichments/registry/sqlquery/Rdbms.scala | 6 +- .../sqlquery/SqlQueryEnrichment.scala | 142 ++--- .../enrichments/web/PageEnrichments.scala | 39 +- .../common/loaders/CljTomcatLoader.scala | 41 +- .../common/loaders/CloudfrontLoader.scala | 105 ++-- .../common/loaders/IpAddressExtractor.scala | 11 +- .../common/loaders/Loader.scala | 85 +-- .../common/loaders/NdjsonLoader.scala | 74 +-- .../common/loaders/ThriftLoader.scala | 62 +- .../common/loaders/TsvLoader.scala | 11 +- .../common/loaders/collectorPayload.scala | 60 +- .../common/outputs/BadRow.scala | 63 +- .../common/utils/ConversionUtils.scala | 256 +++----- .../common/utils/HttpClient.scala | 17 +- .../common/utils/JsonPath.scala | 88 +-- .../common/utils/JsonUtils.scala | 181 +++--- .../common/utils/MapTransformer.scala | 106 ++-- .../common/utils/ScalazCirceUtils.scala | 44 ++ .../common/utils/ScalazJson4sUtils.scala | 71 --- .../common/utils/shredder/Shredder.scala | 184 +++--- .../common/utils/shredder/TypeHierarchy.scala | 58 +- .../package.scala | 96 +-- .../SpecHelpers.scala | 5 +- .../adapters/registry/AdapterSpec.scala | 181 +++--- .../registry/CallrailAdapterSpec.scala | 108 ++-- .../CloudfrontAccessLogAdapterSpec.scala | 131 ++-- .../registry/GoogleAnalyticsAdapterSpec.scala | 110 ++-- .../registry/HubSpotAdapterSpec.scala | 84 ++- .../adapters/registry/IgluAdapterSpec.scala | 295 +++++---- .../registry/MailchimpAdapterSpec.scala | 257 +++++--- .../registry/MailgunAdapterSpec.scala | 268 ++++++--- .../registry/MandrillAdapterSpec.scala | 144 +++-- .../registry/MarketoAdapterSpec.scala | 42 +- .../adapters/registry/OlarkAdapterSpec.scala | 110 ++-- .../registry/PagerdutyAdapterSpec.scala | 131 ++-- .../registry/PingdomAdapterSpec.scala | 80 ++- .../registry/SendgridAdapterSpec.scala | 156 ++--- .../registry/StatusGatorAdapterSpec.scala | 79 ++- .../registry/UnbounceAdapterSpec.scala | 134 +++-- .../registry/UrbanAirshipAdapterSpec.scala | 191 +++--- .../adapters/registry/VeroAdapterSpec.scala | 197 ++++--- .../snowplow/SnowplowAdapterSpec.scala | 252 +++++--- .../enrichments/EnrichmentRegistrySpec.scala | 8 +- .../enrichments/SchemaEnrichmentTest.scala | 68 ++- .../enrichments/clientEnrichmentSpecs.scala | 23 +- .../enrichments/eventEnrichmentSpecs.scala | 58 +- .../enrichments/miscEnrichmentSpecs.scala | 76 +-- .../CampaignAttributionEnrichmentSpec.scala | 83 +-- .../CookieExtractorEnrichmentSpec.scala | 26 +- .../CurrencyConversionEnrichmentSpec.scala | 129 ++-- .../registry/EnrichmentConfigsSpec.scala | 335 ++++++----- .../EventFingerprintEnrichmentSpec.scala | 32 +- .../HttpHeaderExtractorEnrichmentSpec.scala | 23 +- .../registry/IabEnrichmentSpec.scala | 53 +- .../registry/IpLookupsEnrichmentSpec.scala | 25 +- .../JavascriptScriptEnrichmentSpec.scala | 19 +- .../RefererParserEnrichmentSpec.scala | 26 +- .../registry/UaParserEnrichmentSpec.scala | 45 +- .../UserAgentUtilsEnrichmentSpec.scala | 53 +- .../registry/WeatherEnrichmentSpec.scala | 95 +-- .../ApiRequestEnrichmentIntegrationTest.scala | 391 ++++++------ .../apirequest/ApiRequestEnrichmentSpec.scala | 558 +++++++++--------- .../registry/apirequest/CacheSpec.scala | 30 +- .../registry/apirequest/HttpApiSpec.scala | 18 +- .../registry/apirequest/InputSpec.scala | 238 ++++---- .../registry/apirequest/OutputSpec.scala | 37 +- .../pii/PiiPseudonymizerEnrichmentSpec.scala | 432 ++++++++------ .../registry/sqlquery/InputSpec.scala | 242 ++++---- .../registry/sqlquery/OutputSpec.scala | 11 +- .../SqlQueryEnrichmentIntegrationTest.scala | 414 +++++++------ .../sqlquery/SqlQueryEnrichmentSpec.scala | 357 +++++------ .../enrichments/web/ExtractPageUriSpec.scala | 17 +- .../web/ParseCrossDomainSpec.scala | 3 +- .../loaders/CljTomcatLoaderSpec.scala | 260 ++++---- .../loaders/CloudfrontLoaderSpec.scala | 288 ++++----- .../loaders/IpAddressExtractorSpec.scala | 41 +- .../loaders/LoaderSpec.scala | 18 +- .../loaders/NdjsonLoaderSpec.scala | 3 +- .../loaders/ThriftLoaderSpec.scala | 155 ++--- .../loaders/TsvLoaderSpec.scala | 14 +- .../loaders/collectorPayloadSpecs.scala | 13 +- .../utils/JsonPathSpec.scala | 89 ++- .../utils/MapTransformerSpec.scala | 42 +- ...Json4sSpec.scala => ScalazCirceSpec.scala} | 17 +- .../utils/TestResourcesRepositoryRef.scala | 38 +- .../utils/ValidateAndReformatJsonSpec.scala | 48 +- .../utils/conversionUtilsSpecs.scala | 203 ++++--- .../utils/shredder/ShredderSpec.scala | 15 +- .../utils/shredder/TypeHierarchySpec.scala | 38 +- project/CommonDependencies.scala | 20 +- 152 files changed, 7871 insertions(+), 7811 deletions(-) delete mode 100644 modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Serializers.scala create mode 100644 modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala create mode 100644 modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazCirceUtils.scala delete mode 100644 modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazJson4sUtils.scala rename modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/{ScalazJson4sSpec.scala => ScalazCirceSpec.scala} (75%) diff --git a/.scalafmt.conf b/.scalafmt.conf index 6eb841a4d..96ee9fb9c 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,6 +1,6 @@ style = default align = none -maxColumn = 120 +maxColumn = 100 docstrings = JavaDoc optIn.breakChainOnFirstMethodDot = true spaces.afterKeywordBeforeParen = true diff --git a/common.sbt b/common.sbt index b74014efb..7bb7c0b6f 100644 --- a/common.sbt +++ b/common.sbt @@ -18,8 +18,8 @@ lazy val root = project .in(file(".")) .settings( - name := "snowplow-common-enrich", - version := "0.38.0", + name := "snowplow-common-enrich", + version := "0.38.0", description := "Common functionality for enriching raw Snowplow events" ) .settings(BuildSettings.formatting) @@ -47,14 +47,14 @@ lazy val root = project Dependencies.Libraries.yauaa, Dependencies.Libraries.kryo, // Scala + Dependencies.Libraries.circeOptics, + Dependencies.Libraries.circeJackson, Dependencies.Libraries.scalaz7, Dependencies.Libraries.snowplowRawEvent, Dependencies.Libraries.collectorPayload, Dependencies.Libraries.schemaSniffer, Dependencies.Libraries.refererParser, Dependencies.Libraries.maxmindIplookups, - Dependencies.Libraries.json4sJackson, - Dependencies.Libraries.json4sScalaz, Dependencies.Libraries.igluClient, Dependencies.Libraries.scalaUri, Dependencies.Libraries.scalaForex, @@ -67,5 +67,5 @@ lazy val root = project Dependencies.Libraries.scalaCheck, Dependencies.Libraries.scaldingArgs, Dependencies.Libraries.mockito - ) + ) ++ Dependencies.Libraries.circeDeps ) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlException.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlException.scala index 998d199c2..ff0188086 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlException.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlException.scala @@ -16,31 +16,19 @@ import scalaz._ /** * The parent for our ETL-specific exceptions - * - * Note that the SnowPlow ETL does **not** - * use exceptions for control flow - it uses - * Scalaz Validation and ValidationNel objects. - * - * However two types of exception we do support - * are: - * - * 1. FatalEtlException - should always cause - * the ETL to die - * 2. UnexpectedEtlException - ETL may die or - * continue, depending on the ETL config + * Note that the SnowPlow ETL does **not** use exceptions for control flow - it uses Scalaz + * Validation and ValidationNel objects. + * However two types of exception we do support are: + * 1. FatalEtlException - should always cause the ETL to die + * 2. UnexpectedEtlException - ETL may die or continue, depending on the ETL config */ sealed class EtlException(msg: String) extends RuntimeException(msg) /** - * Holds ways of constructing the - * exception message from a Scalaz - * Validation or ValidatioNel. - * - * Mixed into the companion objects - * for the exceptions below. + * Holds ways of constructing the exception message from a Scalaz Validation or ValidatioNel. + * Mixed into the companion objects for the exceptions below. */ trait EtlExceptionConstructors[E <: EtlException] { - // Structured type lets us pass in // a factory to construct our E self: { @@ -48,70 +36,44 @@ trait EtlExceptionConstructors[E <: EtlException] { } => /** - * Alternative constructor for - * the companion object. - * - * Converts a Scalaz - * NonEmptyList[String] into a single - * String error message. - * - * @param errs The list of - * error messages - * @return a new EtlException of - * type E + * Alternative constructor for the companion object. + * Converts a Scalaz NonEmptyList[String] into a single String error message. + * @param errs The list of error messages + * @return a new EtlException of type E */ def apply(errs: NonEmptyList[String]): E = apply(errs.list) /** - * Alternative constructor for - * the companion object. - * - * Converts a List[String] into - * a single String error message. - * - * @param errs The list of - * error messages - * @return a new EtlException of - * type E + * Alternative constructor for the companion object. + * Converts a List[String] into a single String error message. + * @param errs The list of error messages + * @return a new EtlException of type E */ def apply(errs: List[String]): E = fac(formatErrors(errs)) /** - * A helper to format the list of - * error messages. - * - * @param errs The list of error - * messages - * @return a nicely formatted - * error String + * A helper to format the list of error messages. + * @param errs The list of error messages + * @return a nicely formatted error String */ private def formatErrors(errs: List[String]): String = "EtlException Errors:\n - %s".format(errs.mkString("\n - ")) } /** - * Companion object for - * FatalEtlException - * - * Contains an apply() constructor - * which takes a Scalaz - * NonEmptyList[String] - see - * ValidationConstructors trait - * for details. + * Companion object for FatalEtlException + * Contains an apply() constructor which takes a Scalaz NonEmptyList[String] - see + * ValidationConstructors trait for details. */ object FatalEtlException extends EtlExceptionConstructors[FatalEtlException] { val fac = (msg: String) => FatalEtlException(msg) } -/** - * Companion object for - * FatalEtlError - */ +/** Companion object for FatalEtlError */ // TODO: delete when Cascading FailureTrap supports exclusions object FatalEtlError { - def apply(errs: NonEmptyList[String]): FatalEtlError = apply(errs.list) @@ -123,56 +85,26 @@ object FatalEtlError { } /** - * Companion object for - * UnexpectedEtlException - * - * Contains an apply() constructor - * which takes a Scalaz - * NonEmptyList[String] - see - * ValidationConstructors trait - * for details. + * Companion object for UnexpectedEtlException + * Contains an apply() constructor which takes a Scalaz NonEmptyList[String] - see + * ValidationConstructors trait for details. */ object UnexpectedEtlException extends EtlExceptionConstructors[UnexpectedEtlException] { val fac = (msg: String) => UnexpectedEtlException(msg) } /** - * A fatal exception in our ETL. - * - * Will only be thrown if the ETL cannot - * feasibly be run - **do not** try to catch - * it, or a kitten dies. - * - * This should be explicitly excluded from - * Cascading Failure Traps, as soon as they - * support this (Cascading 2.2). + * A fatal exception in our ETL. Will only be thrown if the ETL cannot feasibly be run - **do not** + * try to catch it, or a kitten dies. */ case class FatalEtlException(msg: String) extends EtlException(msg) -/** - * A fatal error in our ETL. - * - * We are using this as a workaround: - * because Cascading cannot yet support - * excluding a specific Exception subclass - * (e.g. FatalEtlException) from a Failure - * Trap, we need to throw an Error instead. - * - * For details see: - * https://groups.google.com/forum/?fromgroups=#!topic/cascading-user/Ld5sg1baOyc - */ -// TODO: delete when Cascading FailureTrap supports exclusions +/** A fatal error in our ETL. */ case class FatalEtlError(msg: String) extends Error(msg) /** - * An unexpected exception in our - * ETL. - * - * Will be thrown in the event of - * an unexpected exception. How to - * handle it will depend on the - * setting of the Continue On - * Unexpected Error? flag passed in - * to the ETL. + * An unexpected exception in our ETL. + * Will be thrown in the event of an unexpected exception. How to handle it will depend on the + * setting of the Continue On Unexpected Error? flag passed in to the ETL. */ case class UnexpectedEtlException(msg: String) extends EtlException(msg) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala index 09eccce7b..3bb61df93 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala @@ -25,42 +25,33 @@ import adapters.AdapterRegistry import enrichments.{EnrichmentManager, EnrichmentRegistry} import outputs.EnrichedEvent -/** - * Expresses the end-to-end event pipeline - * supported by the Scala Common Enrich - * project. - */ +/** Expresses the end-to-end event pipeline supported by the Scala Common Enrich project. */ object EtlPipeline { /** - * A helper method to take a ValidatedMaybeCanonicalInput - * and transform it into a List (possibly empty) of - * ValidatedCanonicalOutputs. - * - * We have to do some unboxing because enrichEvent - * expects a raw CanonicalInput as its argument, not - * a MaybeCanonicalInput. - * + * A helper method to take a ValidatedMaybeCanonicalInput and transform it into a List (possibly + * empty) of ValidatedCanonicalOutputs. + * We have to do some unboxing because enrichEvent expects a raw CanonicalInput as its argument, + * not a MaybeCanonicalInput. * @param adapterRegistry Contains all of the events adapters - * @param enrichmentRegistry Contains configuration for all - * enrichments to apply + * @param enrichmentRegistry Contains configuration for all enrichments to apply * @param etlVersion The ETL version * @param etlTstamp The ETL timestamp * @param input The ValidatedMaybeCanonicalInput - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation - * @return the ValidatedMaybeCanonicalOutput. Thanks to - * flatMap, will include any validation errors - * contained within the ValidatedMaybeCanonicalInput + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation + * @return the ValidatedMaybeCanonicalOutput. Thanks to flatMap, will include any validation + * errors contained within the ValidatedMaybeCanonicalInput */ def processEvents( adapterRegistry: AdapterRegistry, enrichmentRegistry: EnrichmentRegistry, etlVersion: String, etlTstamp: DateTime, - input: ValidatedMaybeCollectorPayload)(implicit resolver: Resolver): List[ValidatedEnrichedEvent] = { - - def flattenToList[A](v: Validated[Option[Validated[NonEmptyList[Validated[A]]]]]): List[Validated[A]] = v match { + input: ValidatedMaybeCollectorPayload)( + implicit resolver: Resolver + ): List[ValidatedEnrichedEvent] = { + def flattenToList[A]( + v: Validated[Option[Validated[NonEmptyList[Validated[A]]]]]): List[Validated[A]] = v match { case Success(Some(Success(nel))) => nel.toList case Success(Some(Failure(f))) => List(f.fail) case Failure(f) => List(f.fail) @@ -80,7 +71,11 @@ object EtlPipeline { } yield for { event <- events - enriched = EnrichmentManager.enrichEvent(enrichmentRegistry, etlVersion, etlTstamp, event) + enriched = EnrichmentManager.enrichEvent( + enrichmentRegistry, + etlVersion, + etlTstamp, + event) } yield enriched flattenToList[EnrichedEvent](e) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala index 47b480395..79b7f418f 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala @@ -19,7 +19,11 @@ import Scalaz._ import loaders.CollectorPayload import registry._ -import registry.snowplow.{Tp1Adapter => SpTp1Adapter, Tp2Adapter => SpTp2Adapter, RedirectAdapter => SpRedirectAdapter} +import registry.snowplow.{ + Tp1Adapter => SpTp1Adapter, + Tp2Adapter => SpTp2Adapter, + RedirectAdapter => SpRedirectAdapter +} /** * The AdapterRegistry lets us convert a CollectorPayload @@ -28,26 +32,26 @@ import registry.snowplow.{Tp1Adapter => SpTp1Adapter, Tp2Adapter => SpTp2Adapter class AdapterRegistry(remoteAdapters: Map[(String, String), RemoteAdapter] = Map.empty) { val adapters: Map[(String, String), Adapter] = Map( - (Vendor.Snowplow, "tp1") -> SpTp1Adapter, - (Vendor.Snowplow, "tp2") -> SpTp2Adapter, - (Vendor.Redirect, "tp2") -> SpRedirectAdapter, - (Vendor.Iglu, "v1") -> IgluAdapter, - (Vendor.Callrail, "v1") -> CallrailAdapter, + (Vendor.Snowplow, "tp1") -> SpTp1Adapter, + (Vendor.Snowplow, "tp2") -> SpTp2Adapter, + (Vendor.Redirect, "tp2") -> SpRedirectAdapter, + (Vendor.Iglu, "v1") -> IgluAdapter, + (Vendor.Callrail, "v1") -> CallrailAdapter, (Vendor.Cloudfront, "wd_access_log") -> CloudfrontAccessLogAdapter.WebDistribution, - (Vendor.Mailchimp, "v1") -> MailchimpAdapter, - (Vendor.Mailgun, "v1") -> MailgunAdapter, - (Vendor.GoogleAnalytics, "v1") -> GoogleAnalyticsAdapter, - (Vendor.Mandrill, "v1") -> MandrillAdapter, - (Vendor.Olark, "v1") -> OlarkAdapter, - (Vendor.Pagerduty, "v1") -> PagerdutyAdapter, - (Vendor.Pingdom, "v1") -> PingdomAdapter, - (Vendor.Sendgrid, "v3") -> SendgridAdapter, - (Vendor.StatusGator, "v1") -> StatusGatorAdapter, - (Vendor.Unbounce, "v1") -> UnbounceAdapter, - (Vendor.UrbanAirship, "v1") -> UrbanAirshipAdapter, - (Vendor.Marketo, "v1") -> MarketoAdapter, - (Vendor.Vero, "v1") -> VeroAdapter, - (Vendor.HubSpot, "v1") -> HubSpotAdapter + (Vendor.Mailchimp, "v1") -> MailchimpAdapter, + (Vendor.Mailgun, "v1") -> MailgunAdapter, + (Vendor.GoogleAnalytics, "v1") -> GoogleAnalyticsAdapter, + (Vendor.Mandrill, "v1") -> MandrillAdapter, + (Vendor.Olark, "v1") -> OlarkAdapter, + (Vendor.Pagerduty, "v1") -> PagerdutyAdapter, + (Vendor.Pingdom, "v1") -> PingdomAdapter, + (Vendor.Sendgrid, "v3") -> SendgridAdapter, + (Vendor.StatusGator, "v1") -> StatusGatorAdapter, + (Vendor.Unbounce, "v1") -> UnbounceAdapter, + (Vendor.UrbanAirship, "v1") -> UrbanAirshipAdapter, + (Vendor.Marketo, "v1") -> MarketoAdapter, + (Vendor.Vero, "v1") -> VeroAdapter, + (Vendor.HubSpot, "v1") -> HubSpotAdapter ) ++ remoteAdapters /** @@ -105,7 +109,8 @@ class AdapterRegistry(remoteAdapters: Map[(String, String), RemoteAdapter] = Map case (Vendor.Redirect, "tp2") => SpRedirectAdapter.toRawEvents(payload) case (Vendor.Iglu, "v1") => IgluAdapter.toRawEvents(payload) case (Vendor.Callrail, "v1") => CallrailAdapter.toRawEvents(payload) - case (Vendor.Cloudfront, "wd_access_log") => CloudfrontAccessLogAdapter.WebDistribution.toRawEvents(payload) + case (Vendor.Cloudfront, "wd_access_log") => + CloudfrontAccessLogAdapter.WebDistribution.toRawEvents(payload) case (Vendor.Mailchimp, "v1") => MailchimpAdapter.toRawEvents(payload) case (Vendor.Mailgun, "v1") => MailgunAdapter.toRawEvents(payload) case (Vendor.GoogleAnalytics, "v1") => GoogleAnalyticsAdapter.toRawEvents(payload) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala index 74fc95082..45009fe1c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala @@ -14,18 +14,17 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry -import scala.util.control.NonFatal - -import com.fasterxml.jackson.core.JsonParseException +import cats.syntax.either._ +import cats.syntax.eq._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.parser._ +import io.circe.syntax._ import org.apache.http.NameValuePair import org.joda.time.{DateTime, DateTimeZone} import org.joda.time.format.DateTimeFormat import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.{JsonUtils => JU} @@ -37,13 +36,11 @@ trait Adapter { SchemaKey("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", "1-0-0").toSchemaUri // The Iglu schema URI for a Snowplow custom contexts - private val Contexts = SchemaKey("com.snowplowanalytics.snowplow", "contexts", "jsonschema", "1-0-1").toSchemaUri + private val Contexts = + SchemaKey("com.snowplowanalytics.snowplow", "contexts", "jsonschema", "1-0-1").toSchemaUri // Signature for a Formatter function - type FormatterFunc = (RawEventParameters) => JObject - - // Needed for json4s default extraction formats - implicit val formats = DefaultFormats + type FormatterFunc = (RawEventParameters) => Json // The encoding type to be used val EventEncType = "UTF-8" @@ -51,221 +48,222 @@ trait Adapter { private val AcceptedQueryParameters = Set("nuid", "aid", "cv", "eid", "ttm", "url") // Datetime format we need to convert timestamps to - val JsonSchemaDateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(DateTimeZone.UTC) + val JsonSchemaDateTimeFormat = + DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(DateTimeZone.UTC) + + private def toStringField(seconds: Long): String = { + val dt: DateTime = new DateTime(seconds * 1000) + JsonSchemaDateTimeFormat.print(dt) + } + + private val longToDateString: Json => Option[Json] = (json: Json) => + json + .as[Long] + .toOption + .map(v => Json.fromString(toStringField(v))) + + private val stringToDateString: Json => Json = (json: Json) => + json + .mapString { v => + Either + .catchNonFatal(toStringField(v.toLong)) + .getOrElse(v) + } /** - * Returns an updated event JSON where - * all of the timestamp fields (tsFieldKey:_) have been - * changed to a valid JsonSchema date-time format - * and the "event":_type field has been removed - * - * @param json The event JSON which we need to - * update values for - * @param eventOpt The event type as an Option[String] - * which we are now going to remove from - * the event JSON - * @param tsFieldKey the key name of the timestamp field - * which will be transformed - * @return the updated JSON with valid date-time - * values in the tsFieldKey fields + * Returns an updated event JSON where all of the timestamp fields (tsFieldKey:_) have been + * changed to a valid JsonSchema date-time format and the "event":_type field has been removed + * @param json The event JSON which we need to update values for + * @param eventOpt The event type as an Option[String] which we are now going to remove from + * the event JSON + * @param tsFieldKey the key name of the timestamp field which will be transformed + * @return the updated JSON with valid date-time values in the tsFieldKey fields */ private[registry] def cleanupJsonEventValues( - json: JValue, + json: Json, eventOpt: Option[(String, String)], - tsFieldKey: String): JValue = { - - def toStringField(seconds: Long): JString = { - val dt: DateTime = new DateTime(seconds * 1000) - JString(JsonSchemaDateTimeFormat.print(dt)) - } - - val j1 = json transformField { - case (k, v) => - if (k == tsFieldKey) { - v match { - case JInt(x) => - try { - (k, toStringField(x.longValue())) - } catch { - case NonFatal(_) => (k, JInt(x)) - } - case JString(x) => - try { - (k, toStringField(x.toLong)) - } catch { - case NonFatal(_) => (k, JString(x)) - } - case x => (k, x) - } - } else (k, v) - } - - eventOpt match { - case Some((keyName, eventType)) => j1 removeField { _ == JField(keyName, eventType) } - case None => j1 - } - } + tsFieldKeys: List[String] + ): Json = + json + .mapObject { obj => + val updatedObj = obj.toMap.map { + case (k, v) if tsFieldKeys.contains(k) && v.isString => (k, stringToDateString(v)) + case (k, v) if tsFieldKeys.contains(k) => (k, longToDateString(v).getOrElse(v)) + case (k, v) if v.isObject => (k, cleanupJsonEventValues(v, eventOpt, tsFieldKeys)) + case (k, v) if v.isArray => (k, cleanupJsonEventValues(v, eventOpt, tsFieldKeys)) + case (k, v) => (k, v) + } + JsonObject( + eventOpt + .map { + case (k1, v1) => + updatedObj.filter { case (k2, v2) => !(k1 == k2 && Json.fromString(v1) === v2) } + } + .getOrElse(updatedObj) + .toList: _* + ) + } + .mapArray(_.map(cleanupJsonEventValues(_, eventOpt, tsFieldKeys))) /** * Converts a CollectorPayload instance into raw events. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * @param payload The CollectorPaylod containing one or more raw events as collected by a + * Snowplow collector + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents /** - * Converts a NonEmptyList of name:value - * pairs into a Map. - * + * Converts a NonEmptyList of name:value pairs into a Map. * @param parameters A NonEmptyList of name:value pairs * @return the name:value pairs in Map form */ - // TODO: can this become private? protected[registry] def toMap(parameters: List[NameValuePair]): Map[String, String] = parameters.map(p => (p.getName -> p.getValue)).toList.toMap /** - * Convenience function to build a simple formatter - * of RawEventParameters. - * - * @param bools A List of keys whose values should be - * processed as boolean-like Strings - * @param ints A List of keys whose values should be - * processed as integer-like Strings - * @param dates If Some, a NEL of keys whose values should - * be treated as date-time-like Strings, which will - * require processing from the specified format - * @return a formatter function which converts - * RawEventParameters into a cleaned JObject + * Convenience function to build a simple formatter of RawEventParameters. + * @param bools A List of keys whose values should be processed as boolean-like Strings + * @param ints A List of keys whose values should be processed as integer-like Strings + * @param dates If Some, a NEL of keys whose values should be treated as date-time-like Strings, + * which will require processing from the specified format + * @return a formatter function which converts RawEventParameters into a cleaned JObject */ protected[registry] def buildFormatter( bools: List[String] = Nil, ints: List[String] = Nil, - dateTimes: JU.DateTimeFields = None): FormatterFunc = { (parameters: RawEventParameters) => - for { - p <- parameters.toList - } yield JU.toJField(p._1, p._2, bools, ints, dateTimes) + dateTimes: JU.DateTimeFields = None + ): FormatterFunc = { (parameters: RawEventParameters) => + val jsons = parameters.toList + .map(p => JU.toJson(p._1, p._2, bools, ints, dateTimes)) + Json.obj(jsons: _*) } /** - * Fabricates a Snowplow unstructured event from - * the supplied parameters. Note that to be a - * valid Snowplow unstructured event, the event - * must contain e, p and tv parameters, so we + * Fabricates a Snowplow unstructured event from the supplied parameters. Note that to be a + * valid Snowplow unstructured event, the event must contain e, p and tv parameters, so we * make sure to set those. - * - * @param tracker The name and version of this - * tracker - * @param parameters The raw-event parameters - * we will nest into the unstructured event - * @param schema The schema key which defines this - * unstructured event as a String - * @param formatter A function to take the raw event - * parameters and turn them into a correctly - * formatted JObject that should pass JSON - * Schema validation - * @param platform The default platform to assign - * the event to - * @return the raw-event parameters for a valid - * Snowplow unstructured event + * @param tracker The name and version of this tracker + * @param parameters The raw-event parameters we will nest into the unstructured event + * @param schema The schema key which defines this unstructured event as a String + * @param formatter A function to take the raw event parameters and turn them into a correctly + * formatted JObject that should pass JSON Schema validation + * @param platform The default platform to assign the event to + * @return the raw-event parameters for a valid Snowplow unstructured event */ protected[registry] def toUnstructEventParams( tracker: String, parameters: RawEventParameters, schema: String, formatter: FormatterFunc, - platform: String): RawEventParameters = { + platform: String + ): RawEventParameters = { val params = formatter(parameters - ("nuid", "aid", "cv", "p")) - val json = compact { - ("schema" -> UnstructEvent) ~ - ("data" -> ( - ("schema" -> schema) ~ - ("data" -> params) - )) - } + val json = Json.obj( + "schema" := UnstructEvent, + "data" := Json.obj( + ("schema" := schema), + ("data" := params) + ) + ) Map( "tv" -> tracker, "e" -> "ue", "p" -> parameters.getOrElse("p", platform), // Required field - "ue_pr" -> json) ++ + "ue_pr" -> json.noSpaces) ++ parameters.filterKeys(AcceptedQueryParameters) } /** - * Creates a Snowplow unstructured event by nesting - * the provided JValue in a self-describing envelope - * for the unstructured event. - * - * @param eventJson The event which we will nest - * into the unstructured event + * Creates a Snowplow unstructured event by nesting the provided JValue in a self-describing + * envelope for the unstructured event. + * @param eventJson The event which we will nest into the unstructured event * @return the self-describing unstructured event */ - protected[registry] def toUnstructEvent(eventJson: JValue): JValue = - ("schema" -> UnstructEvent) ~ - ("data" -> eventJson) + protected[registry] def toUnstructEvent(eventJson: Json): Json = Json.obj( + "schema" := UnstructEvent, + "data" := eventJson + ) /** * Creates a Snowplow custom contexts entity by nesting the provided JValue in a self-describing * envelope for the custom contexts. - * * @param contextJson The context which will be nested into the custom contexts envelope * @return the self-describing custom contexts */ - protected[registry] def toContext(contextJson: JValue): JValue = + protected[registry] def toContext(contextJson: Json): Json = toContexts(List(contextJson)) /** - * Creates a Snowplow custom contexts entity by nesting the provided JValues in a self-describing - * envelope for the custom contexts. - * + * Creates a Snowplow custom contexts entity by nesting the provided JValues in a + * self-describing envelope for the custom contexts. * @param contextJsons The contexts which will be nested into the custom contexts envelope * @return the self-describing custom contexts */ - protected[registry] def toContexts(contextJsons: List[JValue]): JValue = - ("schema" -> Contexts) ~ - ("data" -> contextJsons) + protected[registry] def toContexts(contextJsons: List[Json]): Json = Json.obj( + "schema" := Contexts, + "data" := contextJsons + ) /** - * Fabricates a Snowplow unstructured event from - * the supplied parameters. Note that to be a - * valid Snowplow unstructured event, the event - * must contain e, p and tv parameters, so we + * Fabricates a Snowplow unstructured event from the supplied parameters. Note that to be a + * valid Snowplow unstructured event, the event must contain e, p and tv parameters, so we * make sure to set those. - * - * @param tracker The name and version of this - * tracker - * @param qsParams The query-string parameters - * we will nest into the unstructured event - * @param schema The schema key which defines this - * unstructured event as a String - * @param eventJson The event which we will nest - * into the unstructured event - * @param platform The default platform to assign - * the event to - * @return the raw-event parameters for a valid - * Snowplow unstructured event + * @param tracker The name and version of this tracker + * @param qsParams The query-string parameters we will nest into the unstructured event + * @param schema The schema key which defines this unstructured event as a String + * @param eventJson The event which we will nest into the unstructured event + * @param platform The default platform to assign the event to + * @return the raw-event parameters for a valid Snowplow unstructured event */ protected[registry] def toUnstructEventParams( tracker: String, qsParams: RawEventParameters, schema: String, - eventJson: JValue, + eventJson: JsonObject, platform: String): RawEventParameters = { - val json = compact { - toUnstructEvent( - ("schema" -> schema) ~ - ("data" -> eventJson) - ) - } + val json = toUnstructEvent( + Json.obj( + "schema" := schema, + "data" := eventJson + )).noSpaces + + Map( + "tv" -> tracker, + "e" -> "ue", + "p" -> qsParams.getOrElse("p", platform), // Required field + "ue_pr" -> json) ++ + qsParams.filterKeys(AcceptedQueryParameters) + } + + /** + * Fabricates a Snowplow unstructured event from the supplied parameters. Note that to be a + * valid Snowplow unstructured event, the event must contain e, p and tv parameters, so we + * make sure to set those. + * @param tracker The name and version of this tracker + * @param qsParams The query-string parameters we will nest into the unstructured event + * @param schema The schema key which defines this unstructured event as a String + * @param eventJson The event which we will nest into the unstructured event + * @param platform The default platform to assign the event to + * @return the raw-event parameters for a valid Snowplow unstructured event + */ + protected[registry] def toUnstructEventParams( + tracker: String, + qsParams: RawEventParameters, + schema: String, + eventJson: Json, + platform: String): RawEventParameters = { + + val json = toUnstructEvent( + Json.obj( + "schema" := schema, + "data" := eventJson + )).noSpaces Map( "tv" -> tracker, @@ -277,18 +275,17 @@ trait Adapter { /** * USAGE: Multiple event payloads - * * Processes a list of Validated RawEvents * into a ValidatedRawEvents object. If there * were any Failures in the list we will only * return these. - * - * @param rawEventsList The list of RawEvents that needs - * to be processed - * @return the ValidatedRawEvents which will be comprised - * of either Successful RawEvents or Failures + * @param rawEventsList The list of RawEvents that needs to be processed + * @return the ValidatedRawEvents which will be comprised of either Successful RawEvents + * or Failures */ - protected[registry] def rawEventsListProcessor(rawEventsList: List[Validated[RawEvent]]): ValidatedRawEvents = { + protected[registry] def rawEventsListProcessor( + rawEventsList: List[Validated[RawEvent]] + ): ValidatedRawEvents = { val successes: List[RawEvent] = for { @@ -303,31 +300,27 @@ trait Adapter { (successes, failures) match { case (s :: ss, Nil) => NonEmptyList(s, ss: _*).success // No Failures collected. case (_, f :: fs) => NonEmptyList(f, fs: _*).fail // Some or all are Failures, return these. - case (Nil, Nil) => "List of events is empty (should never happen, not catching empty list properly)".failNel + case (Nil, Nil) => + "List of events is empty (should never happen, not catching empty list properly)".failNel } } /** * USAGE: Single event payloads - * - * Gets the correct Schema URI for the event - * passed from the vendor payload - * - * @param eventOpt An Option[String] which will contain a - * String or None - * @param vendor The vendor we are doing a schema - * lookup for; i.e. MailChimp or PagerDuty - * @param eventSchemaMap A map of event types linked - * to their relevant schema URI's - * @return the schema for the event or a Failure-boxed String - * if we cannot recognize the event type + * Gets the correct Schema URI for the event passed from the vendor payload + * @param eventOpt An Option[String] which will contain a String or None + * @param vendor The vendor we are doing a schema lookup for; i.e. MailChimp or PagerDuty + * @param eventSchemaMap A map of event types linked to their relevant schema URI's + * @return the schema for the event or a Failure-boxed String if we cannot recognize the + * event type */ protected[registry] def lookupSchema( eventOpt: Option[String], vendor: String, eventSchemaMap: Map[String, String]): Validated[String] = eventOpt match { - case None => s"$vendor event failed: type parameter not provided - cannot determine event type".failNel + case None => + s"$vendor event failed: type parameter not provided - cannot determine event type".failNel case Some(eventType) => { eventType match { case et if eventSchemaMap.contains(et) => { @@ -337,7 +330,8 @@ trait Adapter { case Some(schema) => schema.success } } - case "" => s"$vendor event failed: type parameter is empty - cannot determine event type".failNel + case "" => + s"$vendor event failed: type parameter is empty - cannot determine event type".failNel case et => s"$vendor event failed: type parameter [$et] not recognized".failNel } } @@ -345,20 +339,13 @@ trait Adapter { /** * USAGE: Multiple event payloads - * - * Gets the correct Schema URI for the event - * passed from the vendor payload - * - * @param eventOpt An Option[String] which will contain a - * String or None - * @param vendor The vendor we are doing a schema - * lookup for; i.e. MailChimp or PagerDuty - * @param index The index of the event we are trying to - * get a schema URI for - * @param eventSchemaMap A map of event types linked - * to their relevant schema URI's - * @return the schema for the event or a Failure-boxed String - * if we cannot recognize the event type + * Gets the correct Schema URI for the event passed from the vendor payload + * @param eventOpt An Option[String] which will contain a String or None + * @param vendor The vendor we are doing a schema lookup for; i.e. MailChimp or PagerDuty + * @param index The index of the event we are trying to get a schema URI for + * @param eventSchemaMap A map of event types linked to their relevant schema URI's + * @return the schema for the event or a Failure-boxed String if we cannot recognize the + * event type */ protected[registry] def lookupSchema( eventOpt: Option[String], @@ -379,33 +366,28 @@ trait Adapter { } case "" => s"$vendor event at index [$index] failed: type parameter is empty - cannot determine event type".failNel - case et => s"$vendor event at index [$index] failed: type parameter [$et] not recognized".failNel + case et => + s"$vendor event at index [$index] failed: type parameter [$et] not recognized".failNel } } } /** - * Attempts to parse a json string into a JValue - * example: {"p":"app"} becomes JObject(List((p,JString(app)))) - * + * Attempts to parse a json string into a JValue example: + * {"p":"app"} becomes JObject(List((p,JString(app)))) * @param jsonStr The string we want to parse into a JValue - * @return a Validated JValue or a NonEmptyList Failure - * containing a JsonParseException + * @return a Validated JValue or a NonEmptyList Failure containing a parsing exception */ - private[registry] def parseJsonSafe(jsonStr: String): Validated[JValue] = - try { - parse(jsonStr).successNel - } catch { - case e: JsonParseException => { - val exception = JU.stripInstanceEtc(e.toString).orNull - s"Event failed to parse into JSON: [${exception}]".failNel - } + private[registry] def parseJsonSafe(jsonStr: String): Validated[Json] = + parse(jsonStr) match { + case Right(json) => json.successNel + case Left(failure) => s"Event failed to parse into JSON: [${failure.message}]".failNel } + private[registry] val snakeCaseOrDashTokenCapturingRegex = "[_-](\\w)".r /** * Converts dash or unersocre separted strings to camelCase. - * * @param snakeOrDash string like "X-Mailgun-Sid" or "x_mailgun_sid" * @return string like "xMailgunSid" */ @@ -416,12 +398,17 @@ trait Adapter { /** * Converts input field case to camel case recursively - * * @param json parsed event fields as a JValue * @return The mutated event. */ - private[registry] def camelize(json: JValue): JValue = json.mapField { - case (fieldName, JObject(jo)) => (camelCase(fieldName), camelize(jo)) - case (fieldName, jv) => (camelCase(fieldName), jv) - } + private[registry] def camelize(json: Json): Json = + json.asObject match { + case Some(obj) => + Json.obj(obj.toList.map { case (k, v) => (camelCase(k), camelize(v)) }: _*) + case None => + json.asArray match { + case Some(arr) => Json.arr(arr.map(camelize): _*) + case None => json + } + } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala index 04039ee94..5f5abe7e1 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala @@ -21,7 +21,7 @@ import scalaz._ import Scalaz._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} +import utils._ /** * Transforms a collector payload which conforms to @@ -39,13 +39,15 @@ object CallrailAdapter extends Adapter { } // Datetime format used by CallRail (as we will need to massage) - private val CallrailDateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC) + private val CallrailDateTimeFormat = + DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC) // Create a simple formatter function private val CallrailFormatter: FormatterFunc = { val bools = List("first_call", "answered") val ints = List("duration") - val dateTimes: JU.DateTimeFields = Some((NonEmptyList("datetime"), CallrailDateTimeFormat)) + val dateTimes: JsonUtils.DateTimeFields = + Some((NonEmptyList("datetime"), CallrailDateTimeFormat)) buildFormatter(bools, ints, dateTimes) } @@ -70,7 +72,12 @@ object CallrailAdapter extends Adapter { NonEmptyList( RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, params, SchemaUris.CallComplete, CallrailFormatter, "srv"), + parameters = toUnstructEventParams( + TrackerVersion, + params, + SchemaUris.CallComplete, + CallrailFormatter, + "srv"), contentType = payload.contentType, source = payload.source, context = payload.context diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala index 2b6004861..8a22a5828 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala @@ -14,26 +14,22 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry +import scala.util.Try import scala.util.control.NonFatal import com.snowplowanalytics.iglu.client.Resolver +import io.circe._ import org.joda.time.DateTime import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ import loaders.{CollectorContext, CollectorPayload} import utils.ConversionUtils -/** - * Transforms a Cloudfront access log into raw events - */ +/** Transforms a Cloudfront access log into raw events */ object CloudfrontAccessLogAdapter { - /** - * Adapter for Cloudfront web distribution access log files - */ + /** Adapter for Cloudfront web distribution access log files */ object WebDistribution extends Adapter { private val FieldNames = List( @@ -69,12 +65,9 @@ object CloudfrontAccessLogAdapter { /** * Converts a CollectorPayload instance into raw events. - * Chooses a wd_access_log schema version based on the length of the TSV - * Extracts the collector timestamp and IP address from the TSV - * + * Chooses a wd_access_log schema version based on the length of the TSV. + * Extracts the collector timestamp and IP address from the TSV. * @param payload Generated by the TsvLoader. Its body is the raw TSV. - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used * @return a validation boxing either a NEL of raw events or a NEL of failure strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = @@ -89,54 +82,16 @@ object CloudfrontAccessLogAdapter { case 23 => "1-0-4".successNel // 01 Jul 2015 case 24 => "1-0-5".successNel // 29 Sep 2016 case 26 => "1-0-6".successNel - case n => s"Access log TSV line contained $n fields, expected 12, 15, 18, 19, 23, 24 or 26".failNel + case n => + s"Access log TSV line contained $n fields, expected 12, 15, 18, 19, 23, 24 or 26".failNel } schemaVersion.flatMap { v => // Combine the first two fields into a timestamp - val schemaCompatibleFields = "%sT%sZ".format(fields(0), fields(1)) :: fields.toList.tail.tail - - // Attempt to build the json, accumulating errors from unparseable fields - def buildJson( - errors: List[String], - fields: List[(String, String)], - json: JObject): (List[String], JObject) = - fields match { - case Nil => (errors, json) - case head :: tail => - head match { - - case (name, "") => buildJson(errors, tail, json ~ ((name, null))) - case ("timeTaken", field) => - try { - buildJson(errors, tail, json ~ (("timeTaken", field.toDouble))) - } catch { - case e: NumberFormatException => - buildJson( - "Field [timeTaken]: cannot convert [%s] to Double".format(field) :: errors, - tail, - json) - } - case (name, field) if name == "csBytes" || name == "scBytes" => - try { - buildJson(errors, tail, json ~ ((name, field.toInt))) - } catch { - case e: NumberFormatException => - buildJson("Field [%s]: cannot convert [%s] to Int".format(name, field) :: errors, tail, json) - } - case (name, field) if name == "csReferer" || name == "csUserAgent" => - ConversionUtils - .doubleDecode(name, field) - .fold( - e => buildJson(e :: errors, tail, json), - s => buildJson(errors, tail, json ~ ((name, s))) - ) - case ("csUriQuery", field) => - buildJson(errors, tail, json ~ (("csUriQuery", ConversionUtils.singleEncodePcts(field)))) - case (name, field) => buildJson(errors, tail, json ~ ((name, field))) - } - } + val schemaCompatibleFields = + "%sT%sZ".format(fields(0), fields(1)) :: fields.toList.tail.tail - val (errors, ueJson) = buildJson(Nil, FieldNames zip schemaCompatibleFields, JObject()) + val (errors, ueJson) = + buildJson(Nil, FieldNames zip schemaCompatibleFields, JsonObject.empty) val failures = errors match { case Nil => None.successNel @@ -183,14 +138,10 @@ object CloudfrontAccessLogAdapter { } /** - * Converts a CloudFront log-format date and - * a time to a timestamp. - * + * Converts a CloudFront log-format date and a time to a timestamp. * @param date The CloudFront log-format date * @param time The CloudFront log-format time - * @return the timestamp as a Joda DateTime - * or an error String, all wrapped in - * a Scalaz Validation + * @return the timestamp as a Joda DateTime or an error String, all wrapped in a Validation */ def toTimestamp(date: String, time: String): Validation[String, DateTime] = try { @@ -203,5 +154,56 @@ object CloudfrontAccessLogAdapter { .format(date, time, e.getMessage) .fail } + + // Attempt to build the json, accumulating errors from unparseable fields + private def buildJson( + errors: List[String], + fields: List[(String, String)], + json: JsonObject + ): (List[String], JsonObject) = + fields match { + case Nil => (errors, json) + case head :: tail => + head match { + case (name, "") => buildJson(errors, tail, json.add(name, Json.Null)) + case ("timeTaken", field) => + val jsonField = for { + d <- Try(field.toDouble).toOption + js <- Json.fromDouble(d) + } yield js + jsonField match { + case Some(f) => buildJson(errors, tail, json.add("timeTaken", f)) + case None => + buildJson( + "Field [timeTaken]: cannot convert [%s] to Double".format(field) :: errors, + tail, + json + ) + } + case (name, field) if name == "csBytes" || name == "scBytes" => + Try(field.toInt).toOption match { + case Some(i) => buildJson(errors, tail, json.add(name, Json.fromInt(i))) + case None => + buildJson( + "Field [%s]: cannot convert [%s] to Int".format(name, field) :: errors, + tail, + json + ) + } + case (name, field) if name == "csReferer" || name == "csUserAgent" => + ConversionUtils + .doubleDecode(name, field) + .fold( + e => buildJson(e :: errors, tail, json), + s => buildJson(errors, tail, json.add(name, Json.fromString(s))) + ) + case ("csUriQuery", field) => + buildJson( + errors, + tail, + json.add("csUriQuery", Json.fromString(ConversionUtils.singleEncodePcts(field)))) + case (name, field) => buildJson(errors, tail, json.add(name, Json.fromString(field))) + } + } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala index 2e1587a31..873004153 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala @@ -21,12 +21,11 @@ import java.nio.charset.StandardCharsets.UTF_8 import scala.collection.JavaConversions._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.syntax._ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.ConversionUtils._ @@ -74,12 +73,14 @@ object GoogleAnalyticsAdapter extends Adapter { final case class IntType(i: Int) extends FieldType final case class DoubleType(d: Double) extends FieldType final case class BooleanType(b: Boolean) extends FieldType - implicit val fieldTypeJson4s: FieldType => JValue = (f: FieldType) => - f match { - case StringType(s) => JString(s) - case IntType(i) => JInt(i) - case DoubleType(f) => JDouble(f) - case BooleanType(b) => JBool(b) + implicit val encodeFieldType: Encoder[FieldType] = new Encoder[FieldType] { + def apply(f: FieldType): Json = + f match { + case StringType(s) => Json.fromString(s) + case IntType(i) => Json.fromInt(i) + case DoubleType(f) => Json.fromDoubleOrNull(f) + case BooleanType(b) => Json.fromBoolean(b) + } } // translations between string and the needed types in the measurement protocol Iglu schemas @@ -236,7 +237,9 @@ object GoogleAnalyticsAdapter extends Adapter { "fl" -> idTranslation("flashVersion") ) ), - MPData(SchemaKey(Vendor, "link", Format, SchemaVersion), Map("linkid" -> idTranslation("id"))), + MPData( + SchemaKey(Vendor, "link", Format, SchemaVersion), + Map("linkid" -> idTranslation("id"))), MPData( SchemaKey(Vendor, "app", Format, SchemaVersion), Map( @@ -444,8 +447,11 @@ object GoogleAnalyticsAdapter extends Adapter { * @param payload original CollectorPayload * @return a Validation boxing either a RawEvent or a NEL of Failure Strings */ - private def parsePayload(bodyPart: String, payload: CollectorPayload): ValidationNel[String, RawEvent] = { - val params = toMap(URLEncodedUtils.parse(URI.create(s"http://localhost/?$bodyPart"), UTF_8).toList) + private def parsePayload( + bodyPart: String, + payload: CollectorPayload): ValidationNel[String, RawEvent] = { + val params = toMap( + URLEncodedUtils.parse(URI.create(s"http://localhost/?$bodyPart"), UTF_8).toList) params.get("t") match { case None => s"No $VendorName t parameter provided: cannot determine hit type".failNel case Some(hitType) => @@ -455,7 +461,10 @@ object GoogleAnalyticsAdapter extends Adapter { .get(hitType) .map(_.translationTable) .toSuccess(s"No matching $VendorName hit type for hit type $hitType".wrapNel) - val schemaVal = lookupSchema(hitType.some, VendorName, unstructEventData.mapValues(_.schemaKey.toSchemaUri)) + val schemaVal = lookupSchema( + hitType.some, + VendorName, + unstructEventData.mapValues(_.schemaKey.toSchemaUri)) val simpleContexts = buildContexts(params, contextData, fieldToSchemaMap) val compositeContexts = buildCompositeContexts( @@ -478,11 +487,11 @@ object GoogleAnalyticsAdapter extends Adapter { } val contextParam = if (contextJsons.isEmpty) Map.empty - else Map("co" -> compact(toContexts(contextJsons))) + else Map("co" -> toContexts(contextJsons).noSpaces) translatePayload(params, trTable) .map { e => - val unstructEvent = compact(toUnstructEvent(buildJson(schema, e))) + val unstructEvent = toUnstructEvent(buildJson(schema, e)).noSpaces RawEvent( api = payload.api, parameters = contextParam ++ mappings ++ @@ -592,7 +601,9 @@ object GoogleAnalyticsAdapter extends Adapter { composite <- originalParams .filterKeys(k => k.exists(_.isDigit)) .right - brokenDown <- composite.toList.sorted.map { case (k, v) => breakDownCompField(k, v, indicator) }.sequenceU + brokenDown <- composite.toList.sorted.map { + case (k, v) => breakDownCompField(k, v, indicator) + }.sequenceU partitioned = brokenDown.map(_.partition(_._1.startsWith(indicator))).unzip // we additionally make sure we have a rectangular dataset grouped = (partitioned._2 ++ removeConsecutiveDuplicates(partitioned._1)).flatten @@ -621,7 +632,9 @@ object GoogleAnalyticsAdapter extends Adapter { val values = transpose(m.values.map(_.toList).toList) k -> (originalParams.get("cu") match { case Some(currency) if schemasWithCU.contains(k) => - values.map(m.keys zip _).map(l => ("currencyCode" -> StringType(currency) :: l.toList).toMap) + values + .map(m.keys zip _) + .map(l => ("currencyCode" -> StringType(currency) :: l.toList).toMap) case _ => values.map(m.keys zip _).map(_.toMap) }) @@ -681,7 +694,8 @@ object GoogleAnalyticsAdapter extends Adapter { * @param fieldName raw composite field name * @return the break down of the field or a failure if it couldn't be parsed */ - private[registry] def breakDownCompField(fieldName: String): \/[String, (List[String], List[String])] = + private[registry] def breakDownCompField( + fieldName: String): \/[String, (List[String], List[String])] = fieldName match { case compositeFieldRegex(grps @ _*) => splitEvenOdd(grps.toList.filter(_.nonEmpty)).right case s if s.isEmpty => "Cannot parse empty composite field name".left @@ -718,6 +732,9 @@ object GoogleAnalyticsAdapter extends Adapter { case head => head :: transpose(l.collect { case _ :: tail => tail }) } - private def buildJson(schema: String, fields: Map[String, FieldType]): JValue = - ("schema" -> schema) ~ ("data" -> fields) + private def buildJson(schema: String, fields: Map[String, FieldType]): Json = + Json.obj( + "schema" := schema, + "data" := fields + ) } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala index c5ade2553..4fd2af706 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala @@ -14,24 +14,22 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry -import com.fasterxml.jackson.core.JsonParseException +import cats.instances.option._ +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.parser._ import org.joda.time.DateTime import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the HubSpot webhook subscription - * into raw events. + * Transforms a collector payload which conforms to a known version of the HubSpot webhook + * subscription into raw events. */ object HubSpotAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "HubSpot" @@ -55,105 +53,100 @@ object HubSpotAdapter extends Adapter { ) /** - * Converts a CollectorPayload instance into raw events. - * A HubSpot Tracking payload can contain many events in one. - * We expect the type parameter to be 1 of 9 options otherwise - * we have an unsupported event type. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. A HubSpot Tracking payload can contain + * many events in one. We expect the type parameter to be 1 of 9 options otherwise we have an + * unsupported event type. + * @param payload CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} events to process".failNel + case (None, _) => s"Request body is empty: no $VendorName events to process".failNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failNel case (_, Some(ct)) if ct != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failNel - case (Some(body), _) => { - + s"Content type of $ct provided, expected $ContentType for $VendorName".failNel + case (Some(body), _) => payloadBodyToEvents(body) match { case Failure(str) => str.failNel - case Success(list) => { - + case Success(list) => // Create our list of Validated RawEvents val rawEventsList: List[Validated[RawEvent]] = for { (event, index) <- list.zipWithIndex } yield { - - val eventType: Option[String] = (event \ "subscriptionType").extractOpt[String] + val eventType: Option[String] = for { + obj <- event.asObject + subType <- obj("subscriptionType") + subTypeAsString <- subType.asString + } yield subTypeAsString for { schema <- lookupSchema(eventType, VendorName, index, EventSchemaMap) } yield { - val formattedEvent = reformatParameters(event) val qsParams = toMap(payload.querystring) RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, qsParams, schema, formattedEvent, "srv"), + parameters = toUnstructEventParams( + TrackerVersion, + qsParams, + schema, + formattedEvent, + "srv"), contentType = payload.contentType, source = payload.source, context = payload.context ) } } - // Processes the List for Failures and Successes and returns ValidatedRawEvents rawEventsListProcessor(rawEventsList) - } } - } } /** - * Returns a list of JValue events from the - * HubSpot payload - * - * @param body The payload body from the HubSpot - * event - * @return either a Successful List of JValue JSONs - * or a Failure String + * Returns a list of JValue events from the HubSpot payload + * @param body The payload body from the HubSpot event + * @return either a Successful List of JValue JSONs or a Failure String */ - private[registry] def payloadBodyToEvents(body: String): Validation[String, List[JValue]] = - try { - val parsed = parse(body) - parsed match { - case JArray(list) => list.success - case _ => s"Could not resolve ${VendorName} payload into a JSON array of events".fail - } - } catch { - case e: JsonParseException => { - val exception = JU.stripInstanceEtc(e.toString).orNull - s"${VendorName} payload failed to parse into JSON: [${exception}]".fail - } - } + private[registry] def payloadBodyToEvents(body: String): Validation[String, List[Json]] = + Validation.fromEither(for { + b <- parse(body) + .leftMap(e => s"$VendorName payload failed to parse into JSON: [${e.getMessage}]") + a <- b.asArray.toRight(s"Could not resolve $VendorName payload into a JSON array of events") + } yield a.toList) /** - * Returns an updated HubSpot event JSON where - * the "subscriptionType" field is removed - * and "occurredAt" fields' values have been converted - * - * @param json The event JSON which we need to - * update values for + * Returns an updated HubSpot event JSON where the "subscriptionType" field is removed and + * "occurredAt" fields' values have been converted + * @param json The event JSON which we need to update values for * @return the updated JSON with updated fields and values */ - def reformatParameters(json: JValue): JValue = { - - def toStringField(value: Long): JString = { + def reformatParameters(json: Json): Json = { + def toStringField(value: Long): String = { val dt: DateTime = new DateTime(value) - JString(JsonSchemaDateTimeFormat.print(dt)) + JsonSchemaDateTimeFormat.print(dt) } - json removeField { - case ("subscriptionType", JString(s)) => true - case _ => false - } transformField { - case ("occurredAt", JInt(value)) => ("occurredAt", toStringField(value.toLong)) - } + val longToDateString: Kleisli[Option, Json, Json] = Kleisli( + (json: Json) => + json + .as[Long] + .toOption + .map(v => Json.fromString(toStringField(v))) + ) + + val occurredAtKey = "occurredAt" + (for { + jObj <- json.asObject + newValue = jObj.kleisli + .andThen(longToDateString) + .run(occurredAtKey) + res = newValue + .map(v => jObj.add(occurredAtKey, v)) + .getOrElse(jObj) + .remove("subscriptionType") + } yield Json.fromJsonObject(res)).getOrElse(json) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala index b2a9e55c1..4581b9c2b 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala @@ -19,29 +19,26 @@ import java.nio.charset.StandardCharsets.UTF_8 import scala.collection.JavaConversions._ -import com.fasterxml.jackson.core.JsonParseException +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.parser._ +import io.circe.syntax._ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} /** * Transforms a collector payload which either: - * 1. Provides a set of name-value pairs on a GET querystring - * with a &schema={iglu schema uri} parameter. - * 2. Provides a &schema={iglu schema uri} parameter on a POST - * querystring and a set of name-value pairs in the body. + * 1. Provides a set of kv pairs on a GET querystring with a &schema={iglu schema uri} parameter. + * 2. Provides a &schema={iglu schema uri} parameter on a POST querystring and a set of kv pairs in + * the body. * - Formatted as JSON * - Formatted as a Form Body */ object IgluAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Iglu" @@ -52,38 +49,33 @@ object IgluAdapter extends Adapter { private val IgluFormatter: FormatterFunc = buildFormatter() // For defaults /** - * Converts a CollectorPayload instance into raw events. - * Currently we only support a single event Iglu-compatible - * self-describing event passed in on the querystring. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. Currently we only support a single event + * Iglu-compatible self-describing event passed in on the querystring. + * @param payload The CollectorPaylod containing one or more raw events as collected by a Snowplow + * collector + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = { - val params = toMap(payload.querystring) - (params.get("schema"), payload.body, payload.contentType) match { - case (_, Some(body), None) => s"$VendorName event failed: ContentType must be set for a POST payload".failNel - case (None, Some(body), Some(contentType)) => payloadSdJsonToEvent(payload, body, contentType, params) - case (Some(schemaUri), Some(body), Some(contentType)) => payloadToEventWithSchema(payload, schemaUri, params) + case (_, Some(body), None) => + s"$VendorName event failed: ContentType must be set for a POST payload".failNel + case (None, Some(body), Some(contentType)) => + payloadSdJsonToEvent(payload, body, contentType, params) + case (Some(schemaUri), Some(body), Some(contentType)) => + payloadToEventWithSchema(payload, schemaUri, params) case (Some(schemaUri), None, _) => payloadToEventWithSchema(payload, schemaUri, params) - case (_, _, _) => s"$VendorName event failed: is not a sd-json or a valid GET or POST request".failNel + case (_, _, _) => + s"$VendorName event failed: is not a sd-json or a valid GET or POST request".failNel } } // --- SelfDescribingJson Payloads /** - * Processes a potential SelfDescribingJson into a - * validated raw-event. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector + * Processes a potential SelfDescribingJson into a validated raw-event. + * @param payload The CollectorPaylod containing one or more raw events * @param body The extracted body string * @param contentType The extracted contentType string * @param params The raw map of params from the querystring. @@ -92,7 +84,8 @@ object IgluAdapter extends Adapter { payload: CollectorPayload, body: String, contentType: String, - params: Map[String, String]): ValidatedRawEvents = + params: Map[String, String] + ): ValidatedRawEvents = contentType match { case "application/json" => sdJsonBodyToEvent(payload, body, params) case "application/json; charset=utf-8" => sdJsonBodyToEvent(payload, body, params) @@ -100,91 +93,92 @@ object IgluAdapter extends Adapter { } /** - * Processes a potential SelfDescribingJson into a - * validated raw-event. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector + * Processes a potential SelfDescribingJson into a validated raw-event. + * @param payload The CollectorPaylod containing one or more raw events * @param body The extracted body string * @param params The raw map of params from the querystring. */ private[registry] def sdJsonBodyToEvent( payload: CollectorPayload, body: String, - params: Map[String, String]): ValidatedRawEvents = { - - implicit val formats = org.json4s.DefaultFormats - + params: Map[String, String] + ): ValidatedRawEvents = parseJsonSafe(body) match { - case Success(parsed) => { - ((parsed \ "schema").extractOpt[String], (parsed \ "data").extractOpt[JObject]) match { - case (Some(schemaUri), Some(data)) => { + case Success(parsed) => + parsed.asObject.map(obj => (obj("schema").flatMap(_.as[String].toOption), obj("data"))) match { + case Some((Some(schemaUri), Some(data))) => SchemaKey.parse(schemaUri) match { case Failure(procMsg) => procMsg.getMessage.failNel case Success(_) => { NonEmptyList( RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, params, schemaUri, data, "app"), + parameters = + toUnstructEventParams(TrackerVersion, params, schemaUri, data, "app"), contentType = payload.contentType, source = payload.source, context = payload.context - )).success + ) + ).success } } - } - case (None, _) => s"$VendorName event failed: detected SelfDescribingJson but schema key is missing".failNel - case (_, None) => s"$VendorName event failed: detected SelfDescribingJson but data key is missing".failNel + case Some((None, _)) => + s"$VendorName event failed: detected SelfDescribingJson but schema key is missing".failNel + case Some((_, None)) => + s"$VendorName event failed: detected SelfDescribingJson but data key is missing".failNel + case _ => s"$VendorName event failure: could not parse event as json object".failNel } - } case Failure(err) => err.fail } - } // --- Payloads with the Schema in the Query-String /** - * Processes a payload that has the schema field in - * the query-string. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector + * Processes a payload that has the schema field in the query-string. + * @param payload The CollectorPaylod containing one or more raw events * @param schemaUri The schema-uri found * @param params The raw map of params from the querystring. */ private[registry] def payloadToEventWithSchema( payload: CollectorPayload, schemaUri: String, - params: Map[String, String]): ValidatedRawEvents = + params: Map[String, String] + ): ValidatedRawEvents = SchemaKey.parse(schemaUri) match { case Failure(procMsg) => procMsg.getMessage.failNel case Success(_) => (payload.body, payload.contentType) match { - case (None, _) => { + case (None, _) => NonEmptyList( RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, (params - "schema"), schemaUri, IgluFormatter, "app"), + parameters = toUnstructEventParams( + TrackerVersion, + (params - "schema"), + schemaUri, + IgluFormatter, + "app" + ), contentType = payload.contentType, source = payload.source, context = payload.context - )).success - } - case (Some(body), Some(contentType)) => { + ) + ).success + case (Some(body), Some(contentType)) => contentType match { case "application/json" => jsonBodyToEvent(payload, body, schemaUri, params) - case "application/json; charset=utf-8" => jsonBodyToEvent(payload, body, schemaUri, params) - case "application/x-www-form-urlencoded" => formBodyToEvent(payload, body, schemaUri, params) + case "application/json; charset=utf-8" => + jsonBodyToEvent(payload, body, schemaUri, params) + case "application/x-www-form-urlencoded" => + formBodyToEvent(payload, body, schemaUri, params) case _ => "Content type not supported".failNel } - } case (_, None) => "Content type has not been specified".failNel } } /** * Converts a json payload into a single validated event - * * @param body json payload as POST'd by a webhook * @param payload the rest of the payload details * @param schemaUri the schemaUri for the event @@ -195,8 +189,9 @@ object IgluAdapter extends Adapter { payload: CollectorPayload, body: String, schemaUri: String, - params: Map[String, String]): ValidatedRawEvents = { - def buildRawEvent(e: JValue): RawEvent = + params: Map[String, String] + ): ValidatedRawEvents = { + def buildRawEvent(e: Json): RawEvent = RawEvent( api = payload.api, parameters = toUnstructEventParams(TrackerVersion, (params - "schema"), schemaUri, e, "app"), @@ -205,30 +200,29 @@ object IgluAdapter extends Adapter { context = payload.context ) - parseJsonSafe(body) match { - case Success(parsed) => - parsed match { - case a: JArray => - a.arr match { + parse(body) match { + case Right(parsed) => + parsed.asArray match { + case Some(array) => + array.toList match { case h :: t => (NonEmptyList(buildRawEvent(h)) :::> t.map(buildRawEvent)).success - case Nil => s"$VendorName event failed json sanity check: array of events cannot be empty".failNel + case _ => + s"$VendorName event failed json sanity check: array of events cannot be empty".failNel } case _ => - if (parsed.children.isEmpty) { + if (parsed.asObject.fold(true)(_.isEmpty)) { s"$VendorName event failed json sanity check: has no key-value pairs".failNel } else { NonEmptyList(buildRawEvent(parsed)).success } } - case Failure(err) => err.fail + case Left(err) => err.getMessage.failNel } } /** * Converts a form body payload into a single validated event - * - * @param body the form body from the payload as - * POST'd by a webhook + * @param body the form body from the payload as POST'd by a webhook * @param payload the rest of the payload details * @param schemaUri the schemaUri for the event * @param params The query string parameters @@ -238,28 +232,21 @@ object IgluAdapter extends Adapter { payload: CollectorPayload, body: String, schemaUri: String, - params: Map[String, String]): ValidatedRawEvents = - try { - val bodyMap = toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) - val json = compact(render(bodyMap)) - val event = parse(json) + params: Map[String, String] + ): ValidatedRawEvents = { + val bodyMap = + toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) + val event = bodyMap.asJson - NonEmptyList( - RawEvent( - api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, (params - "schema"), schemaUri, event, "srv"), - contentType = payload.contentType, - source = payload.source, - context = payload.context - )).success - } catch { - case e: JsonParseException => { - val exception = JU.stripInstanceEtc(e.toString).orNull - s"${VendorName} event string failed to parse into JSON: [${exception}]".failNel - } - case e: Exception => { - val exception = JU.stripInstanceEtc(e.toString).orNull - s"${VendorName} incorrect event string : [${exception}]".failNel - } - } + NonEmptyList( + RawEvent( + api = payload.api, + parameters = + toUnstructEventParams(TrackerVersion, (params - "schema"), schemaUri, event, "srv"), + contentType = payload.contentType, + source = payload.source, + context = payload.context + ) + ).success + } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala index 7df410382..942df1ccd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala @@ -20,24 +20,21 @@ import java.nio.charset.StandardCharsets.UTF_8 import scala.collection.JavaConversions._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ import org.apache.http.client.utils.URLEncodedUtils import org.joda.time.DateTimeZone import org.joda.time.format.DateTimeFormat import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ import loaders.CollectorPayload import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the Mailchimp Tracking webhook - * into raw events. + * Transforms a collector payload which conforms to a known version of the Mailchimp Tracking + * webhook into raw events. */ object MailchimpAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "MailChimp" @@ -58,40 +55,36 @@ object MailchimpAdapter extends Adapter { ) // Datetime format used by MailChimp (as we will need to massage) - private val MailchimpDateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC) + private val MailchimpDateTimeFormat = + DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC) // Formatter Function to convert RawEventParameters into a merged Json Object private val MailchimpFormatter: FormatterFunc = { (parameters: RawEventParameters) => - mergeJFields(toJFields(parameters)) + mergeJsons(toJsons(parameters)) } /** - * Converts a CollectorPayload instance into raw events. - * An Mailchimp Tracking payload only contains a single event. - * We expect the name parameter to be 1 of 6 options otherwise - * we have an unsupported event type. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. An Mailchimp Tracking payload only + * contains a single event. + * We expect the name parameter to be 1 of 6 options otherwise we have an unsupported event type. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} event to process".failNel + case (None, _) => s"Request body is empty: no $VendorName event to process".failNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failNel case (_, Some(ct)) if ct != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failNel - case (Some(body), _) => { - - val params = toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) + s"Content type of $ct provided, expected $ContentType for $VendorName".failNel + case (Some(body), _) => + val params = toMap( + URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) params.get("type") match { - case None => s"No ${VendorName} type parameter provided: cannot determine event type".failNel - case Some(eventType) => { - + case None => + s"No $VendorName type parameter provided: cannot determine event type".failNel + case Some(eventType) => val allParams = toMap(payload.querystring) ++ reformatParameters(params) for { schema <- lookupSchema(eventType.some, VendorName, EventSchemaMap) @@ -99,36 +92,33 @@ object MailchimpAdapter extends Adapter { NonEmptyList( RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, allParams, schema, MailchimpFormatter, "srv"), + parameters = toUnstructEventParams( + TrackerVersion, + allParams, + schema, + MailchimpFormatter, + "srv"), contentType = payload.contentType, source = payload.source, context = payload.context )) } - } } - } } /** - * Generates a List of JFields from the raw event parameters. - * - * @param parameters The Map of all the parameters - * for this raw event - * @return a (possibly-empty) List of JFields, where each - * JField represents an entry from teh incoming Map + * Generates a List of json fields from the raw event parameters. + * @param parameters The Map of all the parameters for this raw event + * @return a list of fields, where each field represents an entry from the incoming Map */ - private[registry] def toJFields(parameters: RawEventParameters): List[JField] = + private[registry] def toJsons(parameters: RawEventParameters): List[(String, Json)] = for { (k, v) <- parameters.toList - } yield toNestedJField(toKeys(k), v) + } yield toNestedJson(toKeys(k), v) /** - * Returns a NonEmptyList of nested keys from a String representing - * a field from a URI-encoded POST body. - * - * @param formKey The key String that (may) need to be split based on - * the supplied regexp + * Returns a NEL of nested keys from a String representing a field from a URI-encoded POST body. + * @param formKey The key String that (may) need to be split based on the supplied regexp * @return the key or keys as a NonEmptyList of Strings */ private[registry] def toKeys(formKey: String): NonEmptyList[String] = { @@ -137,57 +127,44 @@ object MailchimpAdapter extends Adapter { } /** - * Recursively generates a correct JField, - * working through the supplied NEL of keys. - * - * @param keys The NEL of keys remaining to - * nest into our JObject - * @param value The value we are going to - * finally insert when we run out - * of keys - * @return a JField built from the list of key(s) and - * a value + * Recursively generates a correct json field, working through the supplied NEL of keys. + * @param keys The NEL of keys remaining to nest into our JObject + * @param value The value we are going to finally insert when we run out of keys + * @return a json field built from the list of key(s) and a value */ - private[registry] def toNestedJField(keys: NonEmptyList[String], value: String): JField = + private[registry] def toNestedJson(keys: NonEmptyList[String], value: String): (String, Json) = keys.toList match { - case h1 :: h2 :: t => JField(h1, toNestedJField(NonEmptyList(h2, t: _*), value)) - case h :: Nil => JField(h, JString(value)) + case h1 :: h2 :: t => (h1, Json.obj(toNestedJson(NonEmptyList(h2, t: _*), value))) + case h :: Nil => (h, Json.fromString(value)) // unreachable but can't pattern match on NEL - case _ => JField("", JString(value)) + case _ => ("", Json.fromString(value)) } /** - * Merges a List of possibly overlapping nested JFields together, - * thus: - * - * val a: JField = ("data", JField("nested", JField("more-nested", JField("str", "hi")))) - * val b: JField = ("data", JField("nested", JField("more-nested", JField("num", 42)))) - * => JObject(List((data,JObject(List((nested,JObject(List((more-nested,JObject(List((str,JString(hi)), (num,JInt(42))))))))))))) - * aka {"data":{"nested":{"more-nested":{"str":"hi","num":42}}}} - * - * @param jfields A (possibly-empty) list of JFields which - * need to be merged together - * @return a fully merged JObject from the List of JFields provided, - * or a JObject(Nil) if the List was empty + * Merges a list of possibly overlapping nested json fields together, thus: + * val a = ("data", ("nested", ("more-nested", ("str", "hi")))) + * val b = ("data", ("nested", ("more-nested", ("num", 42)))) + * => {"data":{"nested":{"more-nested":{"str":"hi","num":42}}}} + * @param jfields A (possibly-empty) list of json fields which need to be merged together + * @return a fully merged json from the List of field provided, or json null if the List was empty */ - private[registry] def mergeJFields(jfields: List[JField]): JObject = + private[registry] def mergeJsons(jfields: List[(String, Json)]): Json = jfields match { - case x :: xs => xs.foldLeft(JObject(x))(_ merge JObject(_)) - case Nil => JObject(Nil) + case x :: xs => xs.foldLeft(Json.obj(x))(_ deepMerge Json.obj(_)) + case Nil => Json.Null } /** - * Reformats the date-time stored in the fired_at parameter - * (if found) so that it can pass JSON Schema date-time validation. - * + * Reformats the date-time stored in the fired_at parameter (if found) so that it can pass JSON + * Schema date-time validation. * @param parameters The parameters to be checked for fixing - * @return the event parameters, either with a fixed date-time - * for fired_at if that key was found, or else the original - * parameters + * @return the event parameters, either with a fixed date-time for fired_at if that key was found, + * or else the original parameters */ private[registry] def reformatParameters(parameters: RawEventParameters): RawEventParameters = parameters.get("fired_at") match { - case Some(firedAt) => parameters.updated("fired_at", JU.toJsonSchemaDateTime(firedAt, MailchimpDateTimeFormat)) + case Some(firedAt) => + parameters.updated("fired_at", JU.toJsonSchemaDateTime(firedAt, MailchimpDateTimeFormat)) case None => parameters } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala index e52946037..292fb10cf 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala @@ -18,28 +18,24 @@ import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 import scala.collection.JavaConversions._ -import scala.util.control.NonFatal import scala.util.{Try, Success => TS, Failure => TF} -import com.fasterxml.jackson.core.JsonParseException +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.syntax._ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the StatusGator Tracking webhook - * into raw events. + * Transforms a collector payload which conforms to a known version of the StatusGator Tracking + * webhook into raw events. */ object MailgunAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Mailgun" @@ -62,36 +58,32 @@ object MailgunAdapter extends Adapter { ) /** - * Converts a CollectorPayload instance into raw events. - * - * A Mailgun Tracking payload contains one single event - * in the body of the payload, stored within a HTTP encoded - * string. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * A Mailgun Tracking payload contains one single event in the body of the payload, stored within + * a HTTP encoded string. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} events to process".failureNel + case (None, _) => s"Request body is empty: no $VendorName events to process".failureNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentTypesStr} for ${VendorName}".failureNel + s"Request body provided but content type empty, expected $ContentTypesStr for $VendorName".failureNel case (_, Some(ct)) if !ContentTypes.exists(ct.startsWith(_)) => - s"Content type of ${ct} provided, expected ${ContentTypesStr} for ${VendorName}".failureNel - case (Some(body), _) if (body.isEmpty) => s"${VendorName} event body is empty: nothing to process".failureNel - case (Some(body), Some(ct)) => { + s"Content type of $ct provided, expected $ContentTypesStr for $VendorName".failureNel + case (Some(body), _) if (body.isEmpty) => + s"$VendorName event body is empty: nothing to process".failureNel + case (Some(body), Some(ct)) => val params = toMap(payload.querystring) Try { getBoundary(ct) .map(parseMultipartForm(body, _)) - .getOrElse(toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList)) + .getOrElse( + toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList)) } match { case TF(e) => - s"${VendorName}Adapter could not parse body: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel + val message = JU.stripInstanceEtc(e.getMessage).orNull + s"$VendorName adapter could not parse body: [$message]".failureNel case TS(bodyMap) => bodyMap .get("event") @@ -109,48 +101,55 @@ object MailgunAdapter extends Adapter { TrackerVersion, params, schemaUri, - cleanupJsonEventValues(mEvent, ("event", eventType).some, "timestamp"), + cleanupJsonEventValues( + mEvent, + ("event", eventType).some, + List("timestamp")), "srv"), contentType = payload.contentType, source = payload.source, context = payload.context - )) + ) + ) } - .getOrElse(s"No ${VendorName} event parameter provided: cannot determine event type".failureNel) + .getOrElse( + s"No $VendorName event parameter provided: cannot determine event type".failureNel) } - } } /** * Adds, removes and converts input fields to output fields - * * @param json parsed event fields as a JValue * @return The mutated event. */ - private def mutateMailgunEvent(json: JValue): Validated[JValue] = { - val jsonCamelCase: JValue = camelize(json) - val dropFields: JObject = jsonCamelCase.filterField({ - case (name: String, _) => !(name == "bodyPlain" || name == "attachmentCount") - }) - Try { - (jsonCamelCase \ "attachmentCount") - .extractOpt[String] - .map(ac => ("attachmentCount" -> JInt(ac.toInt)) ~ dropFields) - .getOrElse(dropFields) - } match { - case TS(jvalue) => jvalue.successNel - case TF(e) => - s"${VendorName} event string has unexpected number format: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel + private def mutateMailgunEvent(json: Json): Validated[Json] = { + val attachmentCountKey = "attachmentCount" + val camelCase = camelize(json) + camelCase.asObject match { + case Some(obj) => + val withFilteredFields = obj + .filterKeys(name => !(name == "bodyPlain" || name == attachmentCountKey)) + val attachmentCount = for { + acJson <- obj(attachmentCountKey) + acInt <- acJson.as[Int].toOption + } yield acInt + val finalJsonObject = attachmentCount match { + case Some(ac) => withFilteredFields.add(attachmentCountKey, Json.fromInt(ac)) + case _ => withFilteredFields + } + Json.fromJsonObject(finalJsonObject).successNel + case _ => s"$VendorName event string is not a json object".failureNel } } - private val boundaryRegex = """multipart/form-data.*?boundary=(?:")?([\S ]{0,69})(?: )*(?:")?$""".r + private val boundaryRegex = + """multipart/form-data.*?boundary=(?:")?([\S ]{0,69})(?: )*(?:")?$""".r /** * Returns the boundary parameter for a message of media type multipart/form-data * (https://www.ietf.org/rfc/rfc2616.txt and https://www.ietf.org/rfc/rfc2046.txt) - * - * @param contentType Header field of the form "multipart/form-data; boundary=353d603f-eede-4b49-97ac-724fbc54ea3c" + * @param contentType Header field of the form + * "multipart/form-data; boundary=353d603f-eede-4b49-97ac-724fbc54ea3c" * @return boundary Option[String] */ private def getBoundary(contentType: String): Option[String] = contentType match { @@ -160,19 +159,17 @@ object MailgunAdapter extends Adapter { /** * Rudimentary parsing the form fields of a multipart/form-data into a Map[String, String] - * other fields will be discarded (see https://www.ietf.org/rfc/rfc1867.txt and https://www.ietf.org/rfc/rfc2046.txt). + * other fields will be discarded + * (see https://www.ietf.org/rfc/rfc1867.txt and https://www.ietf.org/rfc/rfc2046.txt). * This parser will only take into account part headers of content-disposition type form-data * and only the parameter name e.g. * Content-Disposition: form-data; anything="notllokingintothis"; name="key" * * value - * * @param body The body of the message * @param boundary String that separates the body parts * @return a map of the form fields and their values (other fields are dropped) */ - private val formDataRegex = - """(?sm).*Content-Disposition:\s*form-data\s*;[ \S\t]*?name="([^"]+)"[ \S\t]*$.*?(?<=^[ \t\S]*$)^\s*(.*?)(?:\s*)\z""".r private def parseMultipartForm(body: String, boundary: String): Map[String, String] = body .split(s"--$boundary") @@ -182,29 +179,18 @@ object MailgunAdapter extends Adapter { }) .toMap + private val formDataRegex = + """(?sm).*Content-Disposition:\s*form-data\s*;[ \S\t]*?name="([^"]+)"[ \S\t]*$.*?(?<=^[ \t\S]*$)^\s*(.*?)(?:\s*)\z""".r + /** * Converts a querystring payload into an event * @param bodyMap The converted map from the querystring */ - private def payloadBodyToEvent(bodyMap: Map[String, String]): Validated[JObject] = + private def payloadBodyToEvent(bodyMap: Map[String, String]): Validated[Json] = (bodyMap.get("timestamp"), bodyMap.get("token"), bodyMap.get("signature")) match { - case (None, _, _) => s"${VendorName} event data missing 'timestamp'".failureNel - case (_, None, _) => s"${VendorName} event data missing 'token'".failureNel - case (_, _, None) => s"${VendorName} event data missing 'signature'".failureNel - case (Some(timestamp), Some(token), Some(signature)) => { - try { - val json = compact(render(bodyMap)) - val event = parse(json) - event match { - case obj: JObject => obj.success - case _ => s"${VendorName} event wrong type: [%s]".format(event.getClass).failureNel - } - } catch { - case e: JsonParseException => - s"${VendorName} event string failed to parse into JSON: [${JU.stripInstanceEtc(e.toString).orNull}]".failureNel - case NonFatal(e) => - s"${VendorName} incorrect event string: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel - } - } + case (None, _, _) => s"$VendorName event data missing 'timestamp'".failureNel + case (_, None, _) => s"$VendorName event data missing 'token'".failureNel + case (_, _, None) => s"$VendorName event data missing 'signature'".failureNel + case (Some(timestamp), Some(token), Some(signature)) => bodyMap.asJson.success } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala index 218b5ca98..6e145cf59 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala @@ -19,24 +19,21 @@ import java.nio.charset.StandardCharsets.UTF_8 import scala.collection.JavaConversions._ -import com.fasterxml.jackson.core.JsonParseException +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.parser._ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the Mandrill Tracking webhook + * Transforms a collector payload which conforms to a known version of the Mandrill Tracking webhook * into raw events. */ object MandrillAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Mandrill" @@ -61,108 +58,87 @@ object MandrillAdapter extends Adapter { /** * Converts a CollectorPayload instance into raw events. - * - * A Mandrill Tracking payload contains many events in - * the body of the payload, stored within a HTTP encoded - * string. - * We expect the event parameter of these events to be - * 1 of 9 options otherwise we have an unsupported event - * type. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * A Mandrill Tracking payload contains many events in the body of the payload, stored within a + * HTTP encoded string. + * We expect the event parameter of these events to be 1 of 9 options otherwise we have an + * unsupported event type. + * @param payload The CollectorPayload containing one or more raw events as collected by a + * Snowplow collector + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} events to process".failNel + case (None, _) => s"Request body is empty: no $VendorName events to process".failNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failNel case (_, Some(ct)) if ct != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failNel - case (Some(body), _) => { - + s"Content type of $ct provided, expected $ContentType for $VendorName".failNel + case (Some(body), _) => payloadBodyToEvents(body) match { case Failure(str) => str.failNel - case Success(list) => { - + case Success(list) => // Create our list of Validated RawEvents val rawEventsList: List[Validated[RawEvent]] = for { (event, index) <- list.zipWithIndex } yield { - - val eventOpt: Option[String] = (event \ "event").extractOpt[String] + val eventOpt = event.hcursor.get[String]("event").toOption for { schema <- lookupSchema(eventOpt, VendorName, index, EventSchemaMap) } yield { - - val formattedEvent = cleanupJsonEventValues(event, eventOpt match { - case Some(x) => ("event", x).some - case None => None - }, "ts") + val formattedEvent = + cleanupJsonEventValues(event, eventOpt.map(("event", _)), List("ts")) val qsParams = toMap(payload.querystring) RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, qsParams, schema, formattedEvent, "srv"), + parameters = toUnstructEventParams( + TrackerVersion, + qsParams, + schema, + formattedEvent, + "srv"), contentType = payload.contentType, source = payload.source, context = payload.context ) } } - // Processes the List for Failures and Successes and returns ValidatedRawEvents rawEventsListProcessor(rawEventsList) - } } - } } /** - * Returns a list of events from the payload - * body of a Mandrill Event. Each event will - * be formatted as an individual JSON of type - * JValue. - * - * NOTE: - * The payload.body string must adhere to UTF-8 - * encoding standards. - * - * @param rawEventString The encoded string - * from the Mandrill payload body - * @return a list of single events formatted as - * json4s JValue JSONs or a Failure String + * Returns a list of events from the payload body of a Mandrill Event. Each event will be + * formatted as an individual JSON. + * NOTE: The payload.body string must adhere to UTF-8 encoding standards. + * @param rawEventString The encoded string from the Mandrill payload body + * @return a list of single events formatted as JSONs or a Failure String */ - private[registry] def payloadBodyToEvents(rawEventString: String): Validation[String, List[JValue]] = { - - val bodyMap = toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + rawEventString), UTF_8).toList) - + private[registry] def payloadBodyToEvents( + rawEventString: String + ): Validation[String, List[Json]] = { + val bodyMap = toMap( + URLEncodedUtils.parse(URI.create("http://localhost/?" + rawEventString), UTF_8).toList) bodyMap match { - case map if map.size != 1 => s"Mapped ${VendorName} body has invalid count of keys: ${map.size}".fail - case map => { + case map if map.size != 1 => + s"Mapped $VendorName body has invalid count of keys: ${map.size}".fail + case map => map.get("mandrill_events") match { - case None => s"Mapped ${VendorName} body does not have 'mandrill_events' as a key".fail - case Some("") => s"${VendorName} events string is empty: nothing to process".fail - case Some(dStr) => { - try { - val parsed = parse(dStr) - parsed match { - case JArray(list) => list.success - case _ => s"Could not resolve ${VendorName} payload into a JSON array of events".fail - } - } catch { - case e: JsonParseException => { - val exception = JU.stripInstanceEtc(e.toString).orNull - s"${VendorName} events string failed to parse into JSON: [${exception}]".fail - } + case None => s"Mapped $VendorName body does not have 'mandrill_events' as a key".fail + case Some("") => s"$VendorName events string is empty: nothing to process".fail + case Some(dStr) => + parse(dStr) match { + case Right(json) => + json.asArray match { + case Some(array) => array.toList.success + case _ => s"Could not resolve $VendorName payload into a JSON array".fail + } + case Left(e) => + s"$VendorName events couldn't be parsed as JSON: [${e.getMessage}]".fail } - } } - } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala index f64c6c58c..ac53cfca8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala @@ -14,26 +14,23 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry -import scala.util.{Failure, Success, Try} - +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.parser._ import org.joda.time.DateTimeZone import org.joda.time.format.DateTimeFormat import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the Marketo webhook - * into raw events. + * Transforms a collector payload which conforms to a known version of the Marketo webhook into raw + * events. */ object MarketoAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Marketo" @@ -49,69 +46,69 @@ object MarketoAdapter extends Adapter { ) // Datetime format used by Marketo - private val MarketoDateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC) + private val MarketoDateTimeFormat = + DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC) + + // Fields containing data which need to be reformatted + private val DateFields = List( + "acquisition_date", + "created_at", + "email_suspended_at", + "last_referred_enrollment", + "last_referred_visit", + "updated_at", + "datetime", + "last_interesting_moment_date" + ) /** - * Returns a validated JSON payload event - * Converts all date-time values to a valid format - * The payload will be validated against marketo "event" schema - * + * Returns a validated JSON payload event. Converts all date-time values to a valid format. + * The payload will be validated against marketo "event" schema. * @param json The JSON payload sent by Marketo * @param payload Rest of the payload details - * @return a validated JSON payload on - * Success, or a NEL + * @return a validated JSON payload on Success, or a NEL */ private def payloadBodyToEvent(json: String, payload: CollectorPayload): Validated[RawEvent] = for { - parsed <- Try(parse(json)) match { - case Success(p) => p.successNel - case Failure(e) => s"$VendorName event failed to parse into JSON: [${e.getMessage}]".failureNel + parsed <- parse(json) match { + case Right(json) => json.successNel + case Left(e) => s"$VendorName event failed to parse into JSON: [${e.getMessage}]".failureNel } - parsedConverted = parsed.transformField { - case ("acquisition_date", JString(value)) => - ("acquisition_date", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - case ("created_at", JString(value)) => - ("created_at", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - case ("email_suspended_at", JString(value)) => - ("email_suspended_at", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - case ("last_referred_enrollment", JString(value)) => - ("last_referred_enrollment", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - case ("last_referred_visit", JString(value)) => - ("last_referred_visit", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - case ("updated_at", JString(value)) => - ("updated_at", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - case ("datetime", JString(value)) => - ("datetime", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - case ("last_interesting_moment_date", JString(value)) => - ("last_interesting_moment_date", JString(JU.toJsonSchemaDateTime(value, MarketoDateTimeFormat))) - } - // The payload doesn't contain a "type" field so we're constraining the eventType to be of type "event" + parsedConverted <- if (parsed.isObject) reformatParameters(parsed).successNel + else s"$VendorName event is not a json object".failureNel + + // The payload doesn't contain a "type" field so we're constraining the eventType to be of + // type "event" eventType = Some("event") schema <- lookupSchema(eventType, VendorName, EventSchemaMap) - params = toUnstructEventParams(TrackerVersion, toMap(payload.querystring), schema, parsedConverted, "srv") + params = toUnstructEventParams( + TrackerVersion, + toMap(payload.querystring), + schema, + parsedConverted, + "srv" + ) rawEvent = RawEvent( api = payload.api, parameters = params, contentType = payload.contentType, source = payload.source, - context = payload.context) + context = payload.context + ) } yield rawEvent /** * Converts a CollectorPayload instance into raw events. - * Marketo event contains no "type" field and since there's only 1 schema the function lookupschema takes the eventType parameter as "event". + * Marketo event contains no "type" field and since there's only 1 schema the function + * lookupschema takes the eventType parameter as "event". * We expect the type parameter to match the supported events, else * we have an unsupported event type. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ - override def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = + override def toRawEvents(payload: CollectorPayload)(implicit r: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { case (None, _) => s"Request body is empty: no $VendorName event to process".failureNel case (Some(body), _) => { @@ -119,4 +116,19 @@ object MarketoAdapter extends Adapter { rawEventsListProcessor(List(event)) } } + + private[registry] def reformatParameters(json: Json): Json = + json.mapObject { obj => + val updatedObj = obj.toMap.map { + case (k, v) if DateFields.contains(k) => + (k, v.mapString { s => + Either + .catchNonFatal(JU.toJsonSchemaDateTime(s, MarketoDateTimeFormat)) + .getOrElse(s) + }) + case (k, v) if v.isObject => (k, reformatParameters(v)) + case (k, v) => (k, v) + } + JsonObject(updatedObj.toList: _*) + } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala index 0619f261d..9f461dca6 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala @@ -17,29 +17,28 @@ package registry import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 -import scala.util.control.NonFatal import scala.collection.JavaConversions._ import scala.util.{Try, Success => TS, Failure => TF} -import com.fasterxml.jackson.core.JsonParseException +import cats.instances.either._ +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.parser._ +import io.circe.optics.JsonPath._ import org.apache.http.client.utils.URLEncodedUtils import org.joda.time.DateTime import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the Olark Tracking webhook + * Transforms a collector payload which conforms to a known version of the Olark Tracking webhook * into raw events. */ object OlarkAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Olark" @@ -56,91 +55,82 @@ object OlarkAdapter extends Adapter { ) /** - * Converts a CollectorPayload instance into raw events. - * - * An Olark Tracking payload contains one single event - * in the body of the payload, stored within a HTTP encoded - * string. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. An Olark Tracking payload contains one + * single event in the body of the payload, stored within a HTTP encoded string. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} events to process".failureNel + case (None, _) => s"Request body is empty: no $VendorName events to process".failureNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failureNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failureNel case (_, Some(ct)) if ct != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failureNel - case (Some(body), _) if (body.isEmpty) => s"${VendorName} event body is empty: nothing to process".failureNel - case (Some(body), _) => { + s"Content type of $ct provided, expected $ContentType for $VendorName".failureNel + case (Some(body), _) if (body.isEmpty) => + s"$VendorName event body is empty: nothing to process".failureNel + case (Some(body), _) => val qsParams = toMap(payload.querystring) - Try { toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) } match { - case TF(e) => s"${VendorName} could not parse body: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel + Try { + toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) + } match { + case TF(e) => + val message = JU.stripInstanceEtc(e.getMessage).orNull + s"$VendorName could not parse body: [$message]".failureNel case TS(bodyMap) => - payloadBodyToEvent(bodyMap).flatMap { - case event => { - val eventType = (event \ "operators") match { - case (JNothing) => Some("offline_message") - case (_) => Some("transcript") - } - lookupSchema(eventType, VendorName, EventSchemaMap).flatMap { - case schema => - transformTimestamps(event).flatMap { - case transformedEvent => - NonEmptyList( - RawEvent( - api = payload.api, - parameters = toUnstructEventParams( - TrackerVersion, - qsParams, - schema, - camelize(transformedEvent), - "srv"), - contentType = payload.contentType, - source = payload.source, - context = payload.context - )).success - } - } + for { + event <- payloadBodyToEvent(bodyMap) + eventType = event.hcursor.get[Json]("operators").toOption match { + case Some(_) => "transcript" + case _ => "offline_message" } - } + schema <- lookupSchema(eventType.some, VendorName, EventSchemaMap) + transformedEvent <- transformTimestamps(event) + } yield + NonEmptyList( + RawEvent( + api = payload.api, + parameters = toUnstructEventParams( + TrackerVersion, + qsParams, + schema, + camelize(transformedEvent), + "srv"), + contentType = payload.contentType, + source = payload.source, + context = payload.context + )) } - } } /** - * Converts all olark timestamps in a parsed transcript or offline_message json object to iso8601 strings - * + * Converts all olark timestamps in a parsed transcript or offline_message json object to iso8601 + * strings * @param json a parsed event * @return JObject the event with timstamps replaced */ - private def transformTimestamps(json: JValue): Validated[JValue] = { - def toMsec(oTs: String): Long = - (oTs.split('.') match { - case Array(sec) => s"${sec}000" - case Array(sec, msec) => s"${sec}${msec.take(3).padTo(3, '0')}" - }).toLong + private def transformTimestamps(json: Json): Validated[Json] = { + def toMsec(oTs: String): Either[String, Long] = + for { + formatted <- oTs.split('.') match { + case Array(sec) => Right(s"${sec}000") + case Array(sec, msec) => Right(s"${sec}${msec.take(3).padTo(3, '0')}") + case _ => Left(s"$VendorName unexpected timestamp format: $oTs") + } + long <- Either.catchNonFatal(formatted.toLong).leftMap(_.getMessage) + } yield long - Try { - json.transformField { - case JField("items", jArray) => - ("items", jArray.transform { - case jo: JObject => - jo.transformField { - case JField("timestamp", JString(value)) => - ("timestamp", JString(JsonSchemaDateTimeFormat.print(new DateTime(toMsec(value))))) - } - }) - }.successNel - } match { - case TF(e) => - s"${VendorName} could not convert timestamps: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel - case TS(s) => s + type EitherString[A] = Either[String, A] + + val modifiedTimestamps: Either[String, Json] = + root.items.each.timestamp.string.modifyF[EitherString] { v => + toMsec(v).map(long => JsonSchemaDateTimeFormat.print(new DateTime(long))) + }(json) + + modifiedTimestamps match { + case Right(json) => json.successNel + case Left(e) => s"$VendorName could not convert timestamps: [$e]".failureNel } } @@ -148,23 +138,15 @@ object OlarkAdapter extends Adapter { * Converts a querystring payload into an event * @param bodyMap The converted map from the querystring */ - private def payloadBodyToEvent(bodyMap: Map[String, String]): Validated[JObject] = + private def payloadBodyToEvent(bodyMap: Map[String, String]): Validated[Json] = bodyMap.get("data") match { - case None => s"${VendorName} event data does not have 'data' as a key".failureNel - case Some("") => s"${VendorName} event data is empty: nothing to process".failureNel - case Some(json) => { - try { - val event = parse(json) - event match { - case obj: JObject => obj.successNel - case _ => s"${VendorName} event wrong type: [%s]".format(event.getClass).failureNel - } - } catch { - case e: JsonParseException => - s"${VendorName} event string failed to parse into JSON: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel - case NonFatal(e) => - s"${VendorName} incorrect event string : [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel + case None => s"$VendorName event data does not have 'data' as a key".failureNel + case Some("") => s"$VendorName event data is empty: nothing to process".failureNel + case Some(json) => + parse(json) match { + case Right(event) => event.successNel + case Left(e) => + s"$VendorName event string failed to parse into JSON: [${e.getMessage}]".failureNel } - } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala index 98141e4c1..925082a8e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala @@ -14,23 +14,20 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} -import com.fasterxml.jackson.core.JsonParseException +import io.circe._ +import io.circe.parser._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the PagerDuty Tracking webhook - * into raw events. + * Transforms a collector payload which conforms to a known version of the PagerDuty Tracking + * webhook into raw events. */ object PagerdutyAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "PagerDuty" @@ -53,47 +50,44 @@ object PagerdutyAdapter extends Adapter { ) /** - * Converts a CollectorPayload instance into raw events. - * A PagerDuty Tracking payload can contain many events in one. - * We expect the type parameter to be 1 of 7 options otherwise - * we have an unsupported event type. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. A PagerDuty Tracking payload can contain + * many events in one. We expect the type parameter to be 1 of 7 options otherwise we have an + * unsupported event type. + * @param payload The CollectorPaylod containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} events to process".failNel + case (None, _) => s"Request body is empty: no $VendorName events to process".failNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failNel case (_, Some(ct)) if ct != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failNel - case (Some(body), _) => { - + s"Content type of $ct provided, expected $ContentType for $VendorName".failNel + case (Some(body), _) => payloadBodyToEvents(body) match { case Failure(str) => str.failNel - case Success(list) => { - + case Success(list) => // Create our list of Validated RawEvents val rawEventsList: List[Validated[RawEvent]] = for { (event, index) <- list.zipWithIndex } yield { - - val eventOpt: Option[String] = (event \ "type").extractOpt[String] + val eventOpt = event.hcursor.downField("type").as[String].toOption for { schema <- lookupSchema(eventOpt, VendorName, index, EventSchemaMap) } yield { - val formattedEvent = reformatParameters(event) val qsParams = toMap(payload.querystring) RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, qsParams, schema, formattedEvent, "srv"), + parameters = toUnstructEventParams( + TrackerVersion, + qsParams, + schema, + formattedEvent, + "srv" + ), contentType = payload.contentType, source = payload.source, context = payload.context @@ -103,77 +97,56 @@ object PagerdutyAdapter extends Adapter { // Processes the List for Failures and Successes and returns ValidatedRawEvents rawEventsListProcessor(rawEventsList) - } } - } } /** - * Returns a list of JValue events from the - * PagerDuty payload - * - * @param body The payload body from the PagerDuty - * event - * @return either a Successful List of JValue JSONs - * or a Failure String + * Returns a list of JValue events from the PagerDuty payload + * @param body The payload body from the PagerDuty event + * @return either a Successful List of JValue JSONs or a Failure String */ - private[registry] def payloadBodyToEvents(body: String): Validation[String, List[JValue]] = - try { - val parsed = parse(body) - (parsed \ "messages") match { - case JArray(list) => list.success - case JNothing => s"${VendorName} payload does not contain the needed 'messages' key".fail - case _ => s"Could not resolve ${VendorName} payload into a JSON array of events".fail - } - } catch { - case e: JsonParseException => { - val exception = JU.stripInstanceEtc(e.toString).orNull - s"${VendorName} payload failed to parse into JSON: [${exception}]".fail - } + private[registry] def payloadBodyToEvents(body: String): Validation[String, List[Json]] = + parse(body) match { + case Right(parsed) => + parsed.hcursor.downField("messages").focus.flatMap(_.asArray) match { + case Some(array) => array.toList.success + case None => s"Could not resolve $VendorName payload into a JSON array of events".fail + } + case Left(e) => s"$VendorName payload failed to parse into JSON: [${e.getMessage}]".fail } /** - * Returns an updated date-time string for - * cases where PagerDuty does not pass a - * '+' or '-' with the date-time. - * + * Returns an updated date-time string for cases where PagerDuty does not pass a '+' or '-' with + * the date-time. * e.g. "2014-11-12T18:53:47 00:00" -> * "2014-11-12T18:53:47+00:00" - * - * @param dt The date-time we need to - * potentially reformat - * @return the date-time which is now - * correctly formatted + * @param dt The date-time we need to potentially reformat + * @return the date-time which is now correctly formatted */ private[registry] def formatDatetime(dt: String): String = dt.replaceAll(" 00:00$", "+00:00") /** - * Returns an updated event JSON where - * all of the fields with a null string - * have been changed to a null value, - * all event types have been trimmed and - * all timestamps have been correctly - * formatted. - * + * Returns an updated event JSON where all of the fields with a null string have been changed to a + * null value, all event types have been trimmed and all timestamps have been correctly formatted. * e.g. "event" -> "null" * "event" -> null - * * e.g. "type" -> "incident.trigger" * "type" -> "trigger" - * - * @param json The event JSON which we need to - * update values within - * @return the updated JSON with valid null values, - * type values and correctly formatted - * date-time strings + * @param json The event JSON which we need to update values within + * @return the updated JSON with valid null values, type values and formatted date-time strings */ - private[registry] def reformatParameters(json: JValue): JValue = - json transformField { - case (key, JString("null")) => (key, JNull) - case ("type", JString(value)) if value.startsWith("incident.") => - ("type", JString(value.replace("incident.", ""))) - case ("created_on", JString(value)) => ("created_on", JString(formatDatetime(value))) - case ("last_status_change_on", JString(value)) => ("last_status_change_on", JString(formatDatetime(value))) + private[registry] def reformatParameters(json: Json): Json = + json.mapObject { obj => + val updatedObj = obj.toMap.map { + case (k, v) if v == Json.fromString("null") => (k, Json.Null) + case ("type", v) if v.isString => ("type", v.mapString(_.replace("incident.", ""))) + case ("created_on", v) if v.isString => ("created_on", v.mapString(formatDatetime)) + case ("last_status_change_on", v) if v.isString => + ("last_status_change_on", v.mapString(formatDatetime)) + case (k, v) if v.isObject => (k, reformatParameters(v)) + case (k, v) => (k, v) + } + JsonObject(updatedObj.toList: _*) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala index b07906f25..d107293e0 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala @@ -16,23 +16,19 @@ package registry import org.apache.http.NameValuePair -import com.fasterxml.jackson.core.JsonParseException +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the Pingdom Tracking webhook + * Transforms a collector payload which conforms to a known version of the Pingdom Tracking webhook * into raw events. */ object PingdomAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Pingdom" @@ -51,36 +47,28 @@ object PingdomAdapter extends Adapter { ) /** - * Converts a CollectorPayload instance into raw events. - * A Pingdom Tracking payload only contains a single event. - * We expect the name parameter to be one of two types otherwise - * we have an unsupported event type. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. A Pingdom Tracking payload only contains + * a single event. We expect the name parameter to be one of two types otherwise we have an + * unsupported event type. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.querystring) match { - case (Nil) => s"${VendorName} payload querystring is empty: nothing to process".failNel - case (qs) => { - + case Nil => s"$VendorName payload querystring is empty: nothing to process".failNel + case qs => reformatMapParams(qs) match { case Failure(f) => f.fail - case Success(s) => { - + case Success(s) => s.get("message") match { case None => - s"${VendorName} payload querystring does not have 'message' as a key: no event to process".failNel - case Some(event) => { - + s"$VendorName payload querystring does not have 'message' as a key".failNel + case Some(event) => for { - parsedEvent <- parseJson(event) + parsedEvent <- parseJsonSafe(event) schema <- { - val eventOpt = (parsedEvent \ "action").extractOpt[String] + val eventOpt = parsedEvent.hcursor.downField("action").as[String].toOption lookupSchema(eventOpt, VendorName, EventSchemaMap) } } yield { @@ -89,43 +77,42 @@ object PingdomAdapter extends Adapter { NonEmptyList( RawEvent( api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, qsParams, schema, formattedEvent, "srv"), + parameters = toUnstructEventParams( + TrackerVersion, + qsParams, + schema, + formattedEvent, + "srv" + ), contentType = payload.contentType, source = payload.source, context = payload.context - )) + ) + ) } - } } - } } - } } /** - * As Pingdom wraps each value in the querystring within: (u'[content]',) - * we need to check that these wrappers have been removed by the Collector - * before we can use them. + * As Pingdom wraps each value in the querystring within: (u'[content]',) we need to check that + * these wrappers have been removed by the Collector before we can use them. * example: p -> (u'app',) becomes p -> app - * - * The expected behavior is that every value will have been cleaned by - * the Collector; however if this is not the case we will not be able - * to process values. - * - * @param params A list of name-value pairs from the querystring of - * the Pingdom payload - * @return a Map of name-value pairs which has been validated as all - * passing the regex extraction or return a NonEmptyList of Failures - * if any could not pass the regex. + * The expected behavior is that every value will have been cleaned by the Collector; however if + * this is not the case we will not be able to process values. + * @param params A list of name-value pairs from the querystring of the Pingdom payload + * @return a Map of name-value pairs which has been validated as all passing the regex extraction + * or return a NonEmptyList of Failures if any could not pass the regex. */ - private[registry] def reformatMapParams(params: List[NameValuePair]): Validated[Map[String, String]] = { + private[registry] def reformatMapParams( + params: List[NameValuePair] + ): Validated[Map[String, String]] = { val formatted = params.map { value => - { - (value.getName, value.getValue) match { - case (k, PingdomValueRegex(v)) => - s"${VendorName} name-value pair [$k -> $v]: Passed regex - Collector is not catching unicode wrappers anymore".failNel - case (k, v) => (k -> v).successNel - } + (value.getName, value.getValue) match { + case (k, PingdomValueRegex(v)) => + (s"$VendorName name-value pair [$k -> $v]: Passed regex - Collector is not catching " + + "unicode wrappers anymore").failNel + case (k, v) => (k -> v).successNel } } @@ -143,40 +130,16 @@ object PingdomAdapter extends Adapter { case (s :: ss, Nil) => (s :: ss).toMap.successNel // No Failures collected. case (_, f :: fs) => NonEmptyList(f, fs: _*).fail // Some Failures, return only those. case (Nil, Nil) => - "Empty parameters list was passed - should never happen: empty querystring is not being caught".failNel + ("Empty parameters list was passed - should never happen: empty " + + "querystring is not being caught").failNel } } /** - * Attempts to parse a json string into a JValue - * example: {"p":"app"} becomes JObject(List((p,JString(app)))) - * - * @param jsonStr The string we want to parse into a JValue - * @return a Validated JValue or a NonEmptyList Failure - * containing a JsonParseException + * Returns an updated Pingdom Event JSON where the "action" field has been removed + * @param json The event JSON which we need to update values for + * @return the updated JSON without the "action" field included */ - private[registry] def parseJson(jsonStr: String): Validated[JValue] = - try { - parse(jsonStr).successNel - } catch { - case e: JsonParseException => { - val exception = JU.stripInstanceEtc(e.toString).orNull - s"${VendorName} event failed to parse into JSON: [${exception}]".failNel - } - } - - /** - * Returns an updated Pingdom Event JSON where - * the "action" field has been removed - * - * @param json The event JSON which we need to - * update values for - * @return the updated JSON without the "action" - * field included - */ - private[registry] def reformatParameters(json: JValue): JValue = - (json \ "action").extractOpt[String] match { - case Some(eventType) => json removeField { _ == JField("action", JString(eventType)) } - case None => json - } + private[registry] def reformatParameters(json: Json): Json = + json.hcursor.downField("action").delete.top.getOrElse(json) } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala index 0935832f1..3a80adf5c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala @@ -38,12 +38,15 @@ import scala.util.control.NonFatal * @param connectionTimeout max duration of each connection attempt * @param readTimeout max duration of read wait time */ -class RemoteAdapter(val remoteUrl: String, val connectionTimeout: Option[Long], val readTimeout: Option[Long]) +class RemoteAdapter( + val remoteUrl: String, + val connectionTimeout: Option[Long], + val readTimeout: Option[Long]) extends Adapter { - val bodyMissingErrorText = "Missing payload body" - val missingEventsErrorText = "Missing events in the response" - val emptyResponseErrorText = "Empty response" + val bodyMissingErrorText = "Missing payload body" + val missingEventsErrorText = "Missing events in the response" + val emptyResponseErrorText = "Empty response" val incompatibleResponseErrorText = "Incompatible response, missing error and events fields" /** @@ -63,15 +66,16 @@ class RemoteAdapter(val remoteUrl: String, val connectionTimeout: Option[Long], case Some(body) if body.nonEmpty => val json = ("contentType" -> payload.contentType) ~ ("queryString" -> toMap(payload.querystring)) ~ - ("headers" -> payload.context.headers) ~ - ("body" -> payload.body) - val request = HttpClient.buildRequest(remoteUrl, - authUser = None, - authPassword = None, - Some(compact(render(json))), - "POST", - connectionTimeout, - readTimeout) + ("headers" -> payload.context.headers) ~ + ("body" -> payload.body) + val request = HttpClient.buildRequest( + remoteUrl, + authUser = None, + authPassword = None, + Some(compact(render(json))), + "POST", + connectionTimeout, + readTimeout) processResponse(payload, HttpClient.getBody(request)) case _ => bodyMissingErrorText.failNel @@ -89,15 +93,15 @@ class RemoteAdapter(val remoteUrl: String, val connectionTimeout: Option[Long], (parse(bodyAsString) \ "error", parse(bodyAsString) \ "events") match { case (JNull, JNull) | (JNothing, JNothing) => incompatibleResponseErrorText.failNel case (error, JNull | JNothing) => error.extract[String].failNel - case (JNull | JNothing, eventsObj) => + case (JNull | JNothing, eventsObj) => val events = eventsObj.extract[List[Map[String, String]]] rawEventsListProcessor(events.map { event => RawEvent( - api = payload.api, - parameters = event, + api = payload.api, + parameters = event, contentType = payload.contentType, - source = payload.source, - context = payload.context + source = payload.source, + context = payload.context ).success }) case _ => s"Unable to parse response: ${bodyAsString}".failNel @@ -107,8 +111,9 @@ class RemoteAdapter(val remoteUrl: String, val connectionTimeout: Option[Long], } catch { case e: MappingException => s"The events field should be List[Map[String, String]], error: ${e} - response: ${bodyAsString}".failNel - case e: JsonParseException => s"Json is not parsable, error: ${e} - response: ${bodyAsString}".failNel - case NonFatal(e) => s"Unexpected error: $e".failNel + case e: JsonParseException => + s"Json is not parsable, error: ${e} - response: ${bodyAsString}".failNel + case NonFatal(e) => s"Unexpected error: $e".failNel } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala index e343718fb..dd682d988 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala @@ -18,23 +18,19 @@ import javax.mail.internet.ContentType import scala.util.Try -import com.fasterxml.jackson.core.JsonParseException +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe.parser._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the Sendgrid Tracking webhook + * Transforms a collector payload which conforms to a known version of the Sendgrid Tracking webhook * into raw events. */ object SendgridAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Sendgrid" @@ -46,92 +42,79 @@ object SendgridAdapter extends Adapter { // Schemas for reverse-engineering a Snowplow unstructured event private val EventSchemaMap = Map( - "processed" -> SchemaKey("com.sendgrid", "processed", "jsonschema", "2-0-0").toSchemaUri, - "dropped" -> SchemaKey("com.sendgrid", "dropped", "jsonschema", "2-0-0").toSchemaUri, - "delivered" -> SchemaKey("com.sendgrid", "delivered", "jsonschema", "2-0-0").toSchemaUri, - "deferred" -> SchemaKey("com.sendgrid", "deferred", "jsonschema", "2-0-0").toSchemaUri, - "bounce" -> SchemaKey("com.sendgrid", "bounce", "jsonschema", "2-0-0").toSchemaUri, - "open" -> SchemaKey("com.sendgrid", "open", "jsonschema", "2-0-0").toSchemaUri, - "click" -> SchemaKey("com.sendgrid", "click", "jsonschema", "2-0-0").toSchemaUri, - "spamreport" -> SchemaKey("com.sendgrid", "spamreport", "jsonschema", "2-0-0").toSchemaUri, - "unsubscribe" -> SchemaKey("com.sendgrid", "unsubscribe", "jsonschema", "2-0-0").toSchemaUri, + "processed" -> SchemaKey("com.sendgrid", "processed", "jsonschema", "2-0-0").toSchemaUri, + "dropped" -> SchemaKey("com.sendgrid", "dropped", "jsonschema", "2-0-0").toSchemaUri, + "delivered" -> SchemaKey("com.sendgrid", "delivered", "jsonschema", "2-0-0").toSchemaUri, + "deferred" -> SchemaKey("com.sendgrid", "deferred", "jsonschema", "2-0-0").toSchemaUri, + "bounce" -> SchemaKey("com.sendgrid", "bounce", "jsonschema", "2-0-0").toSchemaUri, + "open" -> SchemaKey("com.sendgrid", "open", "jsonschema", "2-0-0").toSchemaUri, + "click" -> SchemaKey("com.sendgrid", "click", "jsonschema", "2-0-0").toSchemaUri, + "spamreport" -> SchemaKey("com.sendgrid", "spamreport", "jsonschema", "2-0-0").toSchemaUri, + "unsubscribe" -> SchemaKey("com.sendgrid", "unsubscribe", "jsonschema", "2-0-0").toSchemaUri, "group_unsubscribe" -> SchemaKey("com.sendgrid", "group_unsubscribe", "jsonschema", "2-0-0").toSchemaUri, "group_resubscribe" -> SchemaKey("com.sendgrid", "group_resubscribe", "jsonschema", "2-0-0").toSchemaUri ) /** - * - * Converts a payload into a list of validated events - * Expects a valid json - returns a single failure if one is not present - * + * Converts a payload into a list of validated events. Expects a valid json - returns a single + * failure if one is not present * @param body json payload as POST'd by sendgrid * @param payload the rest of the payload details - * @return a list of validated events, successes will be the corresponding raw events - * failures will contain a non empty list of the reason(s) for the particular event failing + * @return a list of validated events, successes will be the corresponding raw events failures + * will contain a non empty list of the reason(s) for the particular event failing */ - private def payloadBodyToEvents(body: String, payload: CollectorPayload): List[Validated[RawEvent]] = - try { - - val parsed = parse(body) - - if (parsed.children.isEmpty) { - return List(s"$VendorName event failed json sanity check: has no events".failNel) - } - - for ((itm, index) <- parsed.children.zipWithIndex) - yield { - val eventType = (itm \\ "event").extractOpt[String] - val queryString = toMap(payload.querystring) - - lookupSchema(eventType, VendorName, index, EventSchemaMap) map { schema => - { - RawEvent( - api = payload.api, - parameters = toUnstructEventParams( - TrackerVersion, - queryString, - schema, - cleanupJsonEventValues(itm, ("event", eventType.get).some, "timestamp"), - "srv"), - contentType = payload.contentType, - source = payload.source, - context = payload.context - ) + private def payloadBodyToEvents( + body: String, + payload: CollectorPayload + ): List[Validated[RawEvent]] = + parse(body) match { + case Right(json) => + json.asArray match { + case Some(array) => + array.toList.zipWithIndex.map { + case (item, index) => + val eventType = item.hcursor.downField("event").as[String].toOption + val queryString = toMap(payload.querystring) + lookupSchema(eventType, VendorName, index, EventSchemaMap).map { schema => + RawEvent( + api = payload.api, + parameters = toUnstructEventParams( + TrackerVersion, + queryString, + schema, + cleanupJsonEventValues(item, eventType.map(("event", _)), List("timestamp")), + "srv" + ), + contentType = payload.contentType, + source = payload.source, + context = payload.context + ) + } } - } + case None => List(s"$VendorName event is not an array".failNel) } - - } catch { - case e: JsonParseException => { - val exception = JU.stripInstanceEtc(e.toString).orNull - List(s"$VendorName event failed to parse into JSON: [$exception]".failNel) - } + case Left(e) => + List(s"$VendorName event failed to parse into JSON: [${e.getMessage}]".failNel) } /** - * Converts a CollectorPayload instance into raw events. - * A Sendgrid Tracking payload only contains a single event. - * We expect the name parameter to be 1 of 6 options otherwise - * we have an unsupported event type. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used forValidatedRawEvents - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. A Sendgrid Tracking payload only contains + * a single event. We expect the name parameter to be 1 of 6 options otherwise we have an + * unsupported event type. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} event to process".failNel + case (None, _) => s"Request body is empty: no $VendorName event to process".failNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failNel case (_, Some(ct)) if Try(new ContentType(ct).getBaseType).getOrElse(ct) != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failNel - case (Some(body), _) => { + s"Content type of $ct provided, expected $ContentType for $VendorName".failNel + case (Some(body), _) => val events = payloadBodyToEvents(body, payload) rawEventsListProcessor(events) - } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala index f508e65de..9939eb963 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala @@ -20,25 +20,20 @@ import java.nio.charset.StandardCharsets.UTF_8 import scala.collection.JavaConversions._ import scala.util.{Try, Success => TS, Failure => TF} -import com.fasterxml.jackson.core.JsonParseException import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe.syntax._ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the StatusGator Tracking webhook - * into raw events. + * Transforms a collector payload which conforms to a known version of the StatusGator Tracking + * webhook into raw events. */ object StatusGatorAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "StatusGator" @@ -49,53 +44,48 @@ object StatusGatorAdapter extends Adapter { private val ContentType = "application/x-www-form-urlencoded" // Schemas for reverse-engineering a Snowplow unstructured event - private val EventSchema = SchemaKey("com.statusgator", "status_change", "jsonschema", "1-0-0").toSchemaUri + private val EventSchema = + SchemaKey("com.statusgator", "status_change", "jsonschema", "1-0-0").toSchemaUri /** - * Converts a CollectorPayload instance into raw events. - * - * A StatusGator Tracking payload contains one single event - * in the body of the payload, stored within a HTTP encoded - * string. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. A StatusGator Tracking payload contains + * one single event in the body of the payload, stored within a HTTP encoded string. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} events to process".failureNel + case (None, _) => s"Request body is empty: no $VendorName events to process".failureNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failureNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failureNel case (_, Some(ct)) if ct != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failureNel - case (Some(body), _) if (body.isEmpty) => s"${VendorName} event body is empty: nothing to process".failureNel + s"Content type of $ct provided, expected $ContentType for $VendorName".failureNel + case (Some(body), _) if (body.isEmpty) => + s"$VendorName event body is empty: nothing to process".failureNel case (Some(body), _) => { val qsParams = toMap(payload.querystring) Try { toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) } match { case TF(e) => - s"${VendorName} incorrect event string : [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel + val msg = JU.stripInstanceEtc(e.getMessage).orNull + s"$VendorName incorrect event string : [$msg]".failureNel case TS(bodyMap) => - try { - val a: Map[String, String] = bodyMap - val event = parse(compact(render(a))) - NonEmptyList( - RawEvent( - api = payload.api, - parameters = toUnstructEventParams(TrackerVersion, qsParams, EventSchema, camelize(event), "srv"), - contentType = payload.contentType, - source = payload.source, - context = payload.context - )).success - } catch { - case e: JsonParseException => - s"${VendorName} event string failed to parse into JSON: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel - } + NonEmptyList( + RawEvent( + api = payload.api, + parameters = toUnstructEventParams( + TrackerVersion, + qsParams, + EventSchema, + camelize(bodyMap.asJson), + "srv"), + contentType = payload.contentType, + source = payload.source, + context = payload.context + ) + ).success } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala index 1519488b3..f96646187 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala @@ -20,25 +20,21 @@ import java.nio.charset.StandardCharsets.UTF_8 import scala.util.{Try, Success => TS, Failure => TF} import scala.collection.JavaConversions._ -import com.fasterxml.jackson.core.JsonParseException import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.parser._ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the Unbounce Tracking webhook + * Transforms a collector payload which conforms to a known version of the Unbounce Tracking webhook * into raw events. */ object UnbounceAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Unbounce" @@ -56,80 +52,75 @@ object UnbounceAdapter extends Adapter { ) /** - * Converts a CollectorPayload instance into raw events. - * - * An Unbounce Tracking payload contains one single event - * in the body of the payload, stored within a HTTP encoded - * string. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. An Unbounce Tracking payload contains one + * single event in the body of the payload, stored within a HTTP encoded string. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} events to process".failureNel + case (None, _) => s"Request body is empty: no $VendorName events to process".failureNel case (_, None) => - s"Request body provided but content type empty, expected ${ContentType} for ${VendorName}".failureNel + s"Request body provided but content type empty, expected $ContentType for $VendorName".failureNel case (_, Some(ct)) if ct != ContentType => - s"Content type of ${ct} provided, expected ${ContentType} for ${VendorName}".failureNel + s"Content type of $ct provided, expected $ContentType for $VendorName".failureNel case (Some(body), _) => - if (body.isEmpty) s"${VendorName} event body is empty: nothing to process".failureNel + if (body.isEmpty) s"$VendorName event body is empty: nothing to process".failureNel else { val qsParams = toMap(payload.querystring) - Try { toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) } match { + Try { + toMap(URLEncodedUtils.parse(URI.create("http://localhost/?" + body), UTF_8).toList) + } match { case TF(e) => - s"${VendorName} incorrect event string : [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel + val msg = JU.stripInstanceEtc(e.getMessage).orNull + s"$VendorName incorrect event string : [$msg]".failureNel case TS(bodyMap) => - payloadBodyToEvent(bodyMap).flatMap { - case event => { - lookupSchema(Some("form_post"), VendorName, ContextSchema).flatMap { - case schema: String => - toUnstructEventParams(TrackerVersion, qsParams, schema, event, "srv") match { - case unstructEventParams => - NonEmptyList( - RawEvent( - api = payload.api, - parameters = unstructEventParams, - contentType = payload.contentType, - source = payload.source, - context = payload.context - )).success - } - } + payloadBodyToEvent(bodyMap).flatMap { event => + lookupSchema(Some("form_post"), VendorName, ContextSchema).flatMap { schema => + NonEmptyList( + RawEvent( + api = payload.api, + parameters = + toUnstructEventParams(TrackerVersion, qsParams, schema, event, "srv"), + contentType = payload.contentType, + source = payload.source, + context = payload.context + ) + ).success } } } } } - private def payloadBodyToEvent(bodyMap: Map[String, String]): Validated[JValue] = + private def payloadBodyToEvent(bodyMap: Map[String, String]): Validated[Json] = ( bodyMap.get("page_id"), bodyMap.get("page_name"), bodyMap.get("variant"), bodyMap.get("page_url"), - bodyMap.get("data.json")) match { + bodyMap.get("data.json") + ) match { case (None, _, _, _, _) => s"${VendorName} context data missing 'page_id'".failureNel case (_, None, _, _, _) => s"${VendorName} context data missing 'page_name'".failureNel case (_, _, None, _, _) => s"${VendorName} context data missing 'variant'".failureNel case (_, _, _, None, _) => s"${VendorName} context data missing 'page_url'".failureNel - case (_, _, _, _, None) => s"${VendorName} event data does not have 'data.json' as a key".failureNel + case (_, _, _, _, None) => + s"$VendorName event data does not have 'data.json' as a key".failureNel case (_, _, _, _, Some(dataJson)) if dataJson.isEmpty => - s"${VendorName} event data is empty: nothing to process".failureNel - case (Some(pageId), Some(pageName), Some(variant), Some(pageUrl), Some(dataJson)) => { - try { - val event = parse(compact(render(bodyMap - "data.json"))) - camelize( - ("data.json", parse(dataJson)) :: event - .filterField({ case (name: String, _) => name != "data.xml" })).success - } catch { - case e: JsonParseException => - s"${VendorName} event string failed to parse into JSON: [${JU.stripInstanceEtc(e.getMessage).orNull}]".failureNel + s"$VendorName event data is empty: nothing to process".failureNel + case (Some(pageId), Some(pageName), Some(variant), Some(pageUrl), Some(dataJson)) => + val event = (bodyMap - "data.json" - "data.xml").toList + parse(dataJson) match { + case Right(dJs) => + val js = Json + .obj( + ("data.json", dJs) :: event.map { case (k, v) => (k, Json.fromString(v)) }: _* + ) + camelize(js).success + case Left(e) => + s"$VendorName event string failed to parse into JSON: [${e.getMessage}]".failureNel } - } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala index c9827b047..043935e2e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala @@ -14,24 +14,19 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry -import com.fasterxml.jackson.core.JsonParseException - +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe.parser._ import org.joda.time.{DateTime, DateTimeZone} import scalaz.Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -import utils.{JsonUtils => JU} /** - * Transforms a collector payload which conforms to - * a known version of the UrbanAirship Connect API + * Transforms a collector payload which conforms to a known version of the UrbanAirship Connect API * into raw events. */ object UrbanAirshipAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "UrbanAirship" @@ -43,17 +38,24 @@ object UrbanAirshipAdapter extends Adapter { "CLOSE" -> SchemaKey("com.urbanairship.connect", "CLOSE", "jsonschema", "1-0-0").toSchemaUri, "CUSTOM" -> SchemaKey("com.urbanairship.connect", "CUSTOM", "jsonschema", "1-0-0").toSchemaUri, "FIRST_OPEN" -> SchemaKey("com.urbanairship.connect", "FIRST_OPEN", "jsonschema", "1-0-0").toSchemaUri, - "IN_APP_MESSAGE_DISPLAY" -> SchemaKey("com.urbanairship.connect", "IN_APP_MESSAGE_DISPLAY", "jsonschema", "1-0-0").toSchemaUri, + "IN_APP_MESSAGE_DISPLAY" -> SchemaKey( + "com.urbanairship.connect", + "IN_APP_MESSAGE_DISPLAY", + "jsonschema", + "1-0-0" + ).toSchemaUri, "IN_APP_MESSAGE_EXPIRATION" -> SchemaKey( "com.urbanairship.connect", "IN_APP_MESSAGE_EXPIRATION", "jsonschema", - "1-0-0").toSchemaUri, + "1-0-0" + ).toSchemaUri, "IN_APP_MESSAGE_RESOLUTION" -> SchemaKey( "com.urbanairship.connect", "IN_APP_MESSAGE_RESOLUTION", "jsonschema", - "1-0-0").toSchemaUri, + "1-0-0" + ).toSchemaUri, "LOCATION" -> SchemaKey("com.urbanairship.connect", "LOCATION", "jsonschema", "1-0-0").toSchemaUri, "OPEN" -> SchemaKey("com.urbanairship.connect", "OPEN", "jsonschema", "1-0-0").toSchemaUri, "PUSH_BODY" -> SchemaKey("com.urbanairship.connect", "PUSH_BODY", "jsonschema", "1-0-0").toSchemaUri, @@ -67,73 +69,95 @@ object UrbanAirshipAdapter extends Adapter { ) /** - * Converts payload into a single validated event - * Expects a valid json, returns failure if one is not present - * + * Converts payload into a single validated event. Expects a valid json, returns failure if one is + * not present. * @param body_json json payload as a string * @param payload other payload details - * @return a validated event - a success will contain the corresponding RawEvent, failures will - * contain a reason for failure + * @return a validated event - a success is the RawEvent, failures will contain the reasons */ - private def payloadBodyToEvent(body_json: String, payload: CollectorPayload): Validated[RawEvent] = { - + private def payloadBodyToEvent( + bodyJson: String, + payload: CollectorPayload + ): Validated[RawEvent] = { def toTtmFormat(jsonTimestamp: String) = "%d".format(new DateTime(jsonTimestamp).getMillis) - try { + parse(bodyJson) match { + case Right(json) => + val cursor = json.hcursor + val eventType = cursor.get[String]("type").toOption + val trueTs = cursor.get[String]("occurred").toOption + val eid = cursor.get[String]("id").toOption + val collectorTs = cursor.get[String]("processed").toOption + (trueTs |@| eid |@| collectorTs) { (tts, id, cts) => + lookupSchema(eventType, VendorName, EventSchemaMap).map { schema => + RawEvent( + api = payload.api, + parameters = toUnstructEventParams( + TrackerVersion, + toMap(payload.querystring) ++ Map("ttm" -> toTtmFormat(tts), "eid" -> id), + schema, + json, + "srv" + ), + contentType = payload.contentType, + source = payload.source, + context = payload.context.copy(timestamp = Some(new DateTime(cts, DateTimeZone.UTC))) + ) + } + }.getOrElse(s"$VendorName malformed 'occurred', 'id' or 'processed' fields".failNel) + case Left(e) => s"$VendorName event failed to parse into JSON: [${e.getMessage}]".failNel + } + /**try { val parsed = parse(body_json) val eventType = (parsed \ "type").extractOpt[String] - val trueTimestamp = (parsed \ "occurred").extractOpt[String] val eid = (parsed \ "id").extractOpt[String] val collectorTimestamp = (parsed \ "processed").extractOpt[String] - lookupSchema(eventType, VendorName, EventSchemaMap) map { schema => + lookupSchema(eventType, VendorName, EventSchemaMap).map { schema => RawEvent( api = payload.api, parameters = toUnstructEventParams( TrackerVersion, - toMap(payload.querystring) ++ Map("ttm" -> toTtmFormat(trueTimestamp.get), "eid" -> eid.get), + toMap(payload.querystring) ++ Map( + "ttm" -> toTtmFormat(trueTimestamp.get), + "eid" -> eid.get), schema, parsed, "srv"), contentType = payload.contentType, source = payload.source, - context = payload.context.copy(timestamp = Some(new DateTime(collectorTimestamp.get, DateTimeZone.UTC))) + context = payload.context.copy( + timestamp = Some(new DateTime(collectorTimestamp.get, DateTimeZone.UTC))) ) } - } catch { case e: JsonParseException => { val exception = JU.stripInstanceEtc(e.toString).orNull s"$VendorName event failed to parse into JSON: [$exception]".failNel } - } + }**/ } /** - * Converts a CollectorPayload instance into raw events. - * A UrbanAirship connect API payload only contains a single event. - * We expect the name parameter to match the supported events, else + * Converts a CollectorPayload instance into raw events. A UrbanAirship connect API payload only + * contains a single event. We expect the name parameter to match the supported events, else * we have an unsupported event type. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { - case (None, _) => s"Request body is empty: no ${VendorName} event to process".failNel - case (_, Some(ct)) => s"Content type of ${ct} provided, expected None for ${VendorName}".failNel - case (Some(body), _) => { + case (None, _) => s"Request body is empty: no $VendorName event to process".failNel + case (_, Some(ct)) => + s"Content type of $ct provided, expected None for $VendorName".failNel + case (Some(body), _) => val event = payloadBodyToEvent(body, payload) rawEventsListProcessor(List(event)) - } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala index 5c0ebe538..92b4fd432 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala @@ -14,23 +14,15 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry -import scala.util.{Failure, Success, Try} - import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} -import org.joda.time.DateTime +import io.circe._ +import io.circe.parser._ import scalaz.Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload -/** - * Transforms a collector payload which conforms to - * a known version of the Vero webhook - * into raw events. - */ +/** Transforms a collector payload which fits the Vero webhook into raw events. */ object VeroAdapter extends Adapter { - // Vendor name for Failure Message private val VendorName = "Vero" @@ -53,28 +45,37 @@ object VeroAdapter extends Adapter { ) /** - * Converts a payload into a single validated event - * Expects a valid json returns failure if one is not present - * + * Converts a payload into a single validated event. Expects a valid json returns failure if one + * is not present * @param json Payload body that is sent by Vero * @param payload The details of the payload - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ private def payloadBodyToEvent(json: String, payload: CollectorPayload): Validated[RawEvent] = for { - parsed <- Try(parse(json)) match { - case Success(p) => p.successNel - case Failure(e) => s"$VendorName event failed to parse into JSON: [${e.getMessage}]".failureNel + parsed <- parse(json) match { + case Right(p) => p.successNel + case Left(e) => + s"$VendorName event failed to parse into JSON: [${e.getMessage}]".failureNel } - eventType <- Try((parsed \ "type").extract[String]) match { - case Success(et) => et.successNel - case Failure(e) => s"Could not extract type from $VendorName event JSON: [${e.getMessage}]".failureNel + eventType <- parsed.hcursor.get[String]("type") match { + case Right(et) => et.successNel + case Left(e) => + s"Could not extract type from $VendorName event JSON: [${e.getMessage}]".failureNel } - formattedEvent = cleanupJsonEventValues(parsed, ("type", eventType).some, s"${eventType}_at") + formattedEvent = cleanupJsonEventValues( + parsed, + ("type", eventType).some, + List(s"${eventType}_at", "triggered_at") + ) reformattedEvent = reformatParameters(formattedEvent) schema <- lookupSchema(eventType.some, VendorName, EventSchemaMap) - params = toUnstructEventParams(TrackerVersion, toMap(payload.querystring), schema, reformattedEvent, "srv") + params = toUnstructEventParams( + TrackerVersion, + toMap(payload.querystring), + schema, + reformattedEvent, + "srv") rawEvent = RawEvent( api = payload.api, parameters = params, @@ -84,19 +85,16 @@ object VeroAdapter extends Adapter { } yield rawEvent /** - * Converts a CollectorPayload instance into raw events. - * A Vero API payload only contains a single event. - * We expect the type parameter to match the supported events, else - * we have an unsupported event type. - * - * @param payload The CollectorPayload containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. A Vero API payload only contains a single + * event. We expect the type parameter to match the supported events, otherwise we have an + * unsupported event type. + * @param payload The CollectorPayload containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ - override def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = + override def toRawEvents( + payload: CollectorPayload + )(implicit resolver: Resolver): ValidatedRawEvents = (payload.body, payload.contentType) match { case (None, _) => s"Request body is empty: no $VendorName event to process".failureNel case (Some(body), _) => { @@ -106,24 +104,20 @@ object VeroAdapter extends Adapter { } /** - * Returns an updated Vero event JSON where - * the "_tag" field is renamed to "tag" - * and "triggered_at" fields' values have been converted - * - * @param json The event JSON which we need to - * update values for + * Returns an updated Vero event JSON where the "_tags" field is renamed to "tags" + * @param json The event JSON which we need to update values for * @return the updated JSON with updated fields and values */ - def reformatParameters(json: JValue): JValue = { - - def toStringField(value: Long): JString = { - val dt: DateTime = new DateTime(value) - JString(JsonSchemaDateTimeFormat.print(dt)) - } - - json transformField { - case ("_tags", JObject(v)) => ("tags", JObject(v)) - case ("triggered_at", JInt(value)) => ("triggered_at", toStringField(value.toLong * 1000)) + def reformatParameters(json: Json): Json = { + val oldTagsKey = "_tags" + val tagsKey = "tags" + json.mapObject { obj => + val updatedObj = obj.toMap.map { + case (k, v) if k == oldTagsKey => (tagsKey, v) + case (k, v) if v.isObject => (k, reformatParameters(v)) + case (k, v) => (k, v) + } + JsonObject(updatedObj.toList: _*) } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala index 933cb483a..5cc6bc480 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala @@ -15,13 +15,11 @@ package adapters package registry package snowplow -import com.fasterxml.jackson.databind.JsonNode import com.snowplowanalytics.iglu.client.{Resolver, SchemaKey} +import io.circe._ +import io.circe.syntax._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import loaders.CollectorPayload import utils.{JsonUtils => JU} @@ -31,7 +29,6 @@ import utils.{ConversionUtils => CU} * The Redirect Adapter is essentially a pre-processor for * Snowplow Tracker Protocol v2 above (although it doesn't * use the TP2 code above directly). - * * The &u= parameter used for a redirect is converted into * a URI Redirect entity and then either stored as an * unstructured event, added to an existing contexts array @@ -48,24 +45,19 @@ object RedirectAdapter extends Adapter { // Schema for a URI redirect. Could end up being an event or a context // depending on what else is in the payload private object SchemaUris { - val UriRedirect = SchemaKey("com.snowplowanalytics.snowplow", "uri_redirect", "jsonschema", "1-0-0").toSchemaUri + val UriRedirect = + SchemaKey("com.snowplowanalytics.snowplow", "uri_redirect", "jsonschema", "1-0-0").toSchemaUri } /** - * Converts a CollectorPayload instance into raw events. - * Assumes we have a GET querystring with a u parameter - * for the URI redirect and other parameters per the - * Snowplow Tracker Protocol. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. Assumes we have a GET querystring with + * a u parameter for the URI redirect and other parameters per the Snowplow Tracker Protocol. + * @param payload The CollectorPaylod containing one or more raw events as collected by a + * Snowplow collector + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = { - val originalParams = toMap(payload.querystring) if (originalParams.isEmpty) { "Querystring is empty: cannot be a valid URI redirect".failNel @@ -77,7 +69,7 @@ object RedirectAdapter extends Adapter { val newParams = if (originalParams.contains("e")) { // Already have an event so add the URI redirect as a context (more fiddly) - def newCo = Map("co" -> compact(toContext(json))).successNel + def newCo = Map("co" -> toContext(json).noSpaces).successNel (originalParams.get("cx"), originalParams.get("co")) match { case (None, None) => newCo case (None, Some(co)) if co == "" => newCo @@ -86,7 +78,7 @@ object RedirectAdapter extends Adapter { } } else { // Add URI redirect as an unstructured event - Map("e" -> "ue", "ue_pr" -> compact(toUnstructEvent(json))).successNel + Map("e" -> "ue", "ue_pr" -> toUnstructEvent(json).noSpaces).successNel } val fixedParams = Map( @@ -110,40 +102,33 @@ object RedirectAdapter extends Adapter { } /** - * Builds a self-describing JSON representing a - * URI redirect entity. - * + * Builds a self-describing JSON representing a URI redirect entity. * @param uri The URI we are redirecting to - * @return a URI redirect as a self-describing - * JValue + * @return a URI redirect as a self-describing JValue */ - private def buildUriRedirect(uri: String): JValue = - ("schema" -> SchemaUris.UriRedirect) ~ - ("data" -> ( - ("uri" -> uri) - )) + private def buildUriRedirect(uri: String): Json = + Json.obj( + "schema" := SchemaUris.UriRedirect, + "data" := Json.obj("uri" := uri) + ) /** - * Adds a context to an existing non-Base64-encoded - * self-describing contexts stringified JSON. - * - * Does the minimal amount of validation required - * to ensure the context can be safely added, or + * Adds a context to an existing non-Base64-encoded self-describing contexts stringified JSON. + * Does the minimal amount of validation required to ensure the context can be safely added, or * returns a Failure. - * - * @param new The context to add to the - * existing list of contexts - * @param existing The existing contexts as a - * non-Base64-encoded stringified JSON - * @return an updated non-Base64-encoded self- - * describing contexts stringified JSON + * @param new The context to add to the existing list of contexts + * @param existing The existing contexts as a non-Base64-encoded stringified JSON + * @return an updated non-Base64-encoded self-describing contexts stringified JSON */ - private def addToExistingCo(newContext: JValue, existing: String): Validated[String] = + private def addToExistingCo(newContext: Json, existing: String): Validated[String] = for { - node <- JU.extractJson("co|cx", existing).toValidationNel: Validated[JsonNode] - jvalue = fromJsonNode(node) - merged = jvalue merge render("data" -> List(newContext)) - } yield compact(merged) + json <- JU.extractJson("co|cx", existing).toValidationNel: Validated[Json] + merged = json.hcursor + .downField("data") + .withFocus(_.mapArray(newContext +: _)) + .top + .getOrElse(json) + } yield merged.noSpaces /** * Adds a context to an existing Base64-encoded @@ -160,7 +145,7 @@ object RedirectAdapter extends Adapter { * @return an updated non-Base64-encoded self- * describing contexts stringified JSON */ - private def addToExistingCx(newContext: JValue, existing: String): Validated[String] = + private def addToExistingCx(newContext: Json, existing: String): Validated[String] = for { decoded <- CU.decodeBase64Url("cx", existing).toValidationNel: Validated[String] added <- addToExistingCo(newContext, decoded) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala index 719997ae6..b777aa889 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala @@ -21,26 +21,17 @@ import Scalaz._ import loaders.CollectorPayload -/** - * Version 1 of the Tracker Protocol is GET only. - * All data comes in on the querystring. - */ +/** Version 1 of the Tracker Protocol is GET only. All data comes in on the querystring. */ object Tp1Adapter extends Adapter { /** - * Converts a CollectorPayload instance into raw events. - * Tracker Protocol 1 only supports a single event in a - * payload. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation. Not used - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * Converts a CollectorPayload instance into raw events. Tracker Protocol 1 only supports a single + * event in a payload. + * @param payload The CollectorPaylod containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation. Not used + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = { - val params = toMap(payload.querystring) if (params.isEmpty) { "Querystring is empty: no raw event to process".failNel @@ -52,7 +43,8 @@ object Tp1Adapter extends Adapter { contentType = payload.contentType, source = payload.source, context = payload.context - )).success + ) + ).success } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala index ffca54aef..19fd85bc5 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala @@ -29,32 +29,28 @@ import loaders.CollectorPayload import utils.{JsonUtils => JU} /** - * Version 2 of the Tracker Protocol supports GET and POST. Note that - * with POST, data can still be passed on the querystring. + * Version 2 of the Tracker Protocol supports GET and POST. Note that with POST, data can still be + * passed on the querystring. */ object Tp2Adapter extends Adapter { - // Expected content types for a request body private object ContentTypes { - val list = List("application/json", "application/json; charset=utf-8", "application/json; charset=UTF-8") + val list = + List("application/json", "application/json; charset=utf-8", "application/json; charset=UTF-8") val str = list.mkString(", ") } // Request body expected to validate against this JSON Schema - private val PayloadDataSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "payload_data", "jsonschema", 1, 0) + private val PayloadDataSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "payload_data", "jsonschema", 1, 0) /** * Converts a CollectorPayload instance into N raw events. - * - * @param payload The CollectorPaylod containing one or more - * raw events as collected by a Snowplow collector - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation - * @return a Validation boxing either a NEL of RawEvents on - * Success, or a NEL of Failure Strings + * @param payload The CollectorPaylod containing one or more raw events + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation + * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents(payload: CollectorPayload)(implicit resolver: Resolver): ValidatedRawEvents = { - val qsParams = toMap(payload.querystring) // Verify: body + content type set; content type matches expected; body contains expected JSON Schema; body passes schema validation @@ -86,25 +82,19 @@ object Tp2Adapter extends Adapter { } /** - * Converts a JSON Node into a Validated NEL - * of parameters for a RawEvent. The parameters + * Converts a JSON Node into a Validated NEL of parameters for a RawEvent. The parameters * take the form Map[String, String]. - * - * Takes a second set of parameters to merge with - * the generated parameters (the second set takes + * Takes a second set of parameters to merge with the generated parameters (the second set takes * precedence in case of a clash). - * * @param instance The JSON Node to convert - * @param mergeWith A second set of parameters to - * merge (and possibly overwrite) parameters - * from the instance - * @return a NEL of Map[String, String] parameters - * on Succeess, a NEL of Strings on Failure + * @param mergeWith A second set of parameters to merge (and possibly overwrite) parameters + * from the instance + * @return a NEL of Map[String, String] parameters on Succeess, a NEL of Strings on Failure */ private def toParametersNel( instance: JsonNode, - mergeWith: RawEventParameters): Validated[NonEmptyList[RawEventParameters]] = { - + mergeWith: RawEventParameters + ): Validated[NonEmptyList[RawEventParameters]] = { val events: List[List[Validation[String, (String, String)]]] = for { event <- instance.iterator.toList } yield @@ -132,20 +122,19 @@ object Tp2Adapter extends Adapter { case (s :: ss, Nil) => NonEmptyList(s, ss: _*).success // No Failures collected case (s :: ss, f :: fs) => NonEmptyList(f, fs: _*).fail // Some Failures, return those. Should never happen, unless JSON Schema changed - case (Nil, _) => "List of events is empty (should never happen, did JSON Schema change?)".failNel + case (Nil, _) => + "List of events is empty (should never happen, did JSON Schema change?)".failNel } } /** - * Converts a Java Map.Entry containing a JsonNode - * into a (String -> String) parameter. - * + * Converts a Java Map.Entry containing a JsonNode into a (String -> String) parameter. * @param entry The Java Map.Entry to convert - * @return a Validation boxing either our parameter - * on Success, or an error String on Failure. - * + * @return a Validation boxing either our parameter on Success, or an error String on Failure. */ - private def toParameter(entry: JMapEntry[String, JsonNode]): Validation[String, Tuple2[String, String]] = { + private def toParameter( + entry: JMapEntry[String, JsonNode] + ): Validation[String, Tuple2[String, String]] = { val key = entry.getKey val rawValue = entry.getValue @@ -153,33 +142,29 @@ object Tp2Adapter extends Adapter { case Some(txt) => (key, txt).success case None if rawValue.isTextual => s"Value for key ${key} is a null String (should never happen, did Jackson implementation change?)".fail - case _ => s"Value for key ${key} is not a String (should never happen, did JSON Schema change?)".fail + case _ => + s"Value for key ${key} is not a String (should never happen, did JSON Schema change?)".fail } } /** - * Extract the JSON from a String, and - * validate it against the supplied - * JSON Schema. - * - * @param field The name of the field - * containing the JSON instance - * @param schemaCriterion The schema that we - * expected this self-describing - * JSON to conform to + * Extract the JSON from a String, and validate it against the supplied JSON Schema. + * @param field The name of the field containing the JSON instance + * @param schemaCriterion The schema that we expected this self-describing JSON to conform to * @param instance A JSON instance as String - * @param resolver Our implicit Iglu - * Resolver, for schema lookups - * @return an Option-boxed Validation - * containing either a Nel of - * JsonNodes error message on - * Failure, or a singular - * JsonNode on success + * @param resolver Our implicit Iglu Resolver, for schema lookups + * @return an Option-boxed Validation containing either a Nel of JsonNodes error message on + * Failure, or a singular JsonNode on success */ - private def extractAndValidateJson(field: String, schemaCriterion: SchemaCriterion, instance: String)( - implicit resolver: Resolver): Validated[JsonNode] = + private def extractAndValidateJson( + field: String, + schemaCriterion: SchemaCriterion, + instance: String + )( + implicit resolver: Resolver + ): Validated[JsonNode] = for { - j <- (JU.extractJson(field, instance).toValidationNel: Validated[JsonNode]) + j <- (JU.extractJsonNode(field, instance).toValidationNel: Validated[JsonNode]) v <- j.verifySchemaAndValidate(schemaCriterion, true).leftMap(_.map(_.toString)) } yield v diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala index 44d2c8e76..4a7922339 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala @@ -21,48 +21,36 @@ import scalaz._ import Scalaz._ /** - * Contains enrichments related to the - * client - where the client is the - * software which is using the SnowPlow - * tracker. - * - * Enrichments relate to browser resolution + * Contains enrichments related to the client - where the client is the software which is using the + * Snowplow tracker. Enrichments relate to browser resolution. */ object ClientEnrichments { /** - * The Tracker Protocol's pattern - * for a screen resolution - for - * details see: - * + * The Tracker Protocol's pattern for a screen resolution - for details see: * https://github.com/snowplow/snowplow/wiki/snowplow-tracker-protocol#wiki-browserandos */ private val ResRegex = """(\d+)x(\d+)""".r /** - * Extracts view dimensions (e.g. screen resolution, - * browser/app viewport) stored as per the Tracker - * Protocol: - * + * Extracts view dimensions (e.g. screen resolution, browser/app viewport) stored as per the + * Tracker Protocol: * https://github.com/snowplow/snowplow/wiki/snowplow-tracker-protocol#wiki-browserandos - * - * @param field The name of the field - * holding the screen dimensions - * @param res The packed string - * holding the screen dimensions - * @return the ResolutionTuple or an - * error message, boxed in a - * Scalaz Validation + * @param field The name of the field holding the screen dimensions + * @param res The packed string holding the screen dimensions + * @return the ResolutionTuple or an error message, boxed in a Scalaz Validation */ - val extractViewDimensions: (String, String) => Validation[String, ViewDimensionsTuple] = (field, res) => - res match { - case ResRegex(width, height) => - try { - (width.toInt: JInteger, height.toInt: JInteger).success - } catch { - case NonFatal(e) => "Field [%s]: view dimensions [%s] exceed Integer's max range".format(field, res).fail - } - case _ => "Field [%s]: [%s] does not contain valid view dimensions".format(field, res).fail - } + val extractViewDimensions: (String, String) => Validation[String, (JInteger, JInteger)] = + (field, res) => + res match { + case ResRegex(width, height) => + try { + (width.toInt: JInteger, height.toInt: JInteger).success + } catch { + case NonFatal(e) => + "Field [%s]: view dimensions [%s] exceed Integer's max range".format(field, res).fail + } + case _ => "Field [%s]: [%s] does not contain valid view dimensions".format(field, res).fail + } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala index 5f9dad6fe..822738d31 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala @@ -32,9 +32,7 @@ import outputs.EnrichedEvent /** * A module to hold our enrichment process. - * - * At the moment this is very fixed - no - * support for configuring enrichments etc. + * At the moment this is very fixed - no support for configuring enrichments etc. */ object EnrichmentManager { @@ -43,20 +41,18 @@ object EnrichmentManager { /** * Runs our enrichment process. - * - * @param registry Contains configuration - * for all enrichments to apply + * @param registry Contains configuration for all enrichments to apply * @param hostEtlVersion ETL version * @param etlTstamp ETL timestamp - * @param raw Our canonical input - * to enrich - * @return a MaybeCanonicalOutput - i.e. - * a ValidationNel containing - * either failure Strings or a - * NonHiveOutput. + * @param raw Our canonical input to enrich + * @return a MaybeCanonicalOutput - i.e. a ValidationNel containing either failure Strings */ - def enrichEvent(registry: EnrichmentRegistry, hostEtlVersion: String, etlTstamp: DateTime, raw: RawEvent)( - implicit resolver: Resolver): ValidatedEnrichedEvent = { + def enrichEvent( + registry: EnrichmentRegistry, + hostEtlVersion: String, + etlTstamp: DateTime, + raw: RawEvent + )(implicit resolver: Resolver): ValidatedEnrichedEvent = { // Placeholders for where the Success value doesn't matter. // Useful when you're updating large (>22 field) POSOs. val unitSuccess = ().success[String] @@ -73,7 +69,10 @@ object EnrichmentManager { e.v_etl = ME.etlVersion(hostEtlVersion) e.etl_tstamp = EE.toTimestamp(etlTstamp) e.network_userid = raw.context.userId.orNull // May be updated later by 'nuid' - e.user_ipaddress = ME.extractIp("user_ipaddress", raw.context.ipAddress.orNull).toOption.orNull // May be updated later by 'ip' + e.user_ipaddress = ME + .extractIp("user_ipaddress", raw.context.ipAddress.orNull) + .toOption + .orNull // May be updated later by 'ip' e } @@ -536,7 +535,8 @@ object EnrichmentManager { (customContexts, unstructEvent) match { case (Success(cctx), Success(ue)) => enrichment.lookup(event, preparedDerivedContexts, cctx, ue) - case _ => Nil.success // Skip. Unstruct event or custom context corrupted (event enrichment will fail anyway) + case _ => + Nil.success // Skip. Unstruct event or custom context corrupted (event enrichment will fail anyway) } case None => Nil.success } @@ -547,7 +547,8 @@ object EnrichmentManager { (customContexts, unstructEvent, sqlQueryContexts) match { case (Success(cctx), Success(ue), Success(sctx)) => enrichment.lookup(event, preparedDerivedContexts ++ sctx, cctx, ue) - case _ => Nil.success // Skip. Unstruct event or custom context corrupted (event enrichment will fail anyway) + case _ => + Nil.success // Skip. Unstruct event or custom context corrupted (event enrichment will fail anyway) } case None => Nil.success } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala index 34bdc0491..51b7a915f 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala @@ -18,54 +18,45 @@ import java.net.URI import com.snowplowanalytics.iglu.client.{Resolver, SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ValidatableJsonMethods._ import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ +import io.circe.jackson._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import registry._ import registry.apirequest.{ApiRequestEnrichment, ApiRequestEnrichmentConfig} import registry.pii.PiiPseudonymizerEnrichment import registry.sqlquery.{SqlQueryEnrichment, SqlQueryEnrichmentConfig} -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Companion which holds a constructor - * for the EnrichmentRegistry. - */ +/** Companion which holds a constructor for the EnrichmentRegistry. */ object EnrichmentRegistry { - implicit val formats = DefaultFormats - private val EnrichmentConfigSchemaCriterion = SchemaCriterion("com.snowplowanalytics.snowplow", "enrichments", "jsonschema", 1, 0) /** - * Constructs our EnrichmentRegistry - * from the supplied JSON JValue. - * + * Constructs our EnrichmentRegistry from the supplied JSON JValue. * @param node A JValue representing an array of enrichment JSONs - * @param localMode Whether to use the local MaxMind data file - * Enabled for tests - * @param resolver (implicit) The Iglu resolver used for - * schema lookup and validation - * @return Validation boxing an EnrichmentRegistry object - * containing enrichments configured from node - * @todo remove all the JsonNode round-tripping when - * we have ValidatableJValue + * @param localMode Whether to use the local MaxMind data file, enabled for tests + * @param resolver (implicit) The Iglu resolver used for schema lookup and validation + * @return Validation boxing an EnrichmentRegistry object containing enrichments configured from + * node + * @todo remove all the JsonNode round-tripping when we have ValidatableJValue */ - def parse(node: JValue, localMode: Boolean)(implicit resolver: Resolver): ValidatedNelMessage[EnrichmentRegistry] = { - + def parse(node: Json, localMode: Boolean)( + implicit resolver: Resolver): ValidatedNelMessage[EnrichmentRegistry] = { // Check schema, validate against schema, convert to List[JValue] - val enrichments: ValidatedNelMessage[List[JValue]] = for { - d <- asJsonNode(node).verifySchemaAndValidate(EnrichmentConfigSchemaCriterion, true) + val enrichments: ValidatedNelMessage[List[Json]] = for { + d <- circeToJackson(node).verifySchemaAndValidate(EnrichmentConfigSchemaCriterion, true) } yield - (fromJsonNode(d) match { - case JArray(x) => x + jacksonToCirce(d).asArray match { + case Some(array) => array.toList case _ => throw new Exception( - "Enrichments JSON not an array - the enrichments JSON schema should prevent this happening") - }) + "Enrichments JSON not an array - the enrichments JSON schema should" + + " prevent this happening") + } // Check each enrichment validates against its own schema val configs: ValidatedNelMessage[EnrichmentMap] = (for { @@ -75,8 +66,8 @@ object EnrichmentRegistry { json <- jsons } yield for { - pair <- asJsonNode(json).validateAndIdentifySchema(dataOnly = true) - conf <- buildEnrichmentConfig(pair._1, fromJsonNode(pair._2), localMode) + pair <- circeToJackson(json).validateAndIdentifySchema(dataOnly = true) + conf <- buildEnrichmentConfig(pair._1, jacksonToCirce(pair._2), localMode) } yield conf) .flatMap(_.sequenceU) // Swap nested List[scalaz.Validation[...] .map(_.flatten.toMap) // Eliminate our Option boxing (drop Nones) @@ -86,149 +77,132 @@ object EnrichmentRegistry { } /** - * Builds an Enrichment from a JValue if it has a - * recognized name field and matches a schema key - * + * Builds an Enrichment from a Json if it has a recognized name field and matches a schema key * @param enrichmentConfig JValue with enrichment information * @param schemaKey SchemaKey for the JValue - * @param localMode Whether to use the local MaxMind data file - * Enabled for tests - * @return ValidatedNelMessage boxing Option boxing Tuple2 containing - * the Enrichment object and the schemaKey + * @param localMode Whether to use the local MaxMind data file, enabled for tests + * @return ValidatedNelMessage boxing Option boxing Tuple2 containing the Enrichment object and + * the schemaKey */ private def buildEnrichmentConfig( schemaKey: SchemaKey, - enrichmentConfig: JValue, + enrichmentConfig: Json, localMode: Boolean ): ValidatedNelMessage[Option[Tuple2[String, Enrichment]]] = { - val enabled = ScalazJson4sUtils.extract[Boolean](enrichmentConfig, "enabled").toValidationNel + + val enabled = ScalazCirceUtils.extract[Boolean](enrichmentConfig, "enabled").toValidationNel + enabled match { case Success(false) => None.success.toValidationNel // Enrichment is disabled - case _ => - val name = ScalazJson4sUtils.extract[String](enrichmentConfig, "name").toValidationNel - name.flatMap(nm => - nm match { - case "ip_lookups" => - IpLookupsEnrichment.parse(enrichmentConfig, schemaKey, localMode).map((nm, _).some) - case "anon_ip" => - AnonIpEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "referer_parser" => - RefererParserEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "campaign_attribution" => - CampaignAttributionEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "user_agent_utils_config" => - UserAgentUtilsEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "ua_parser_config" => - UaParserEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "yauaa_enrichment_config" => - YauaaEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "currency_conversion_config" => - CurrencyConversionEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "javascript_script_config" => - JavascriptScriptEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "event_fingerprint_config" => - EventFingerprintEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "cookie_extractor_config" => - CookieExtractorEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "http_header_extractor_config" => - HttpHeaderExtractorEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "weather_enrichment_config" => - WeatherEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "api_request_enrichment_config" => - ApiRequestEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "sql_query_enrichment_config" => - SqlQueryEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "pii_enrichment_config" => - PiiPseudonymizerEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) - case "iab_spiders_and_robots_enrichment" => - IabEnrichment.parse(enrichmentConfig, schemaKey, localMode).map((nm, _).some) - case _ => - None.success.toValidationNel // Enrichment is not recognized yet + case e => { + val name = ScalazCirceUtils.extract[String](enrichmentConfig, "name").toValidationNel + name.flatMap(nm => { + + if (nm == "ip_lookups") { + IpLookupsEnrichment.parse(enrichmentConfig, schemaKey, localMode).map((nm, _).some) + } else if (nm == "anon_ip") { + AnonIpEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "referer_parser") { + RefererParserEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "campaign_attribution") { + CampaignAttributionEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "user_agent_utils_config") { + UserAgentUtilsEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "ua_parser_config") { + UaParserEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "yauaa_enrichment_config") { + YauaaEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "currency_conversion_config") { + CurrencyConversionEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "javascript_script_config") { + JavascriptScriptEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "event_fingerprint_config") { + EventFingerprintEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "cookie_extractor_config") { + CookieExtractorEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "http_header_extractor_config") { + HttpHeaderExtractorEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "weather_enrichment_config") { + WeatherEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "api_request_enrichment_config") { + ApiRequestEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "sql_query_enrichment_config") { + SqlQueryEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "pii_enrichment_config") { + PiiPseudonymizerEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some) + } else if (nm == "iab_spiders_and_robots_enrichment") { + IabEnrichment.parse(enrichmentConfig, schemaKey, localMode).map((nm, _).some) + } else { + None.success // Enrichment is not recognized yet + } }) + } } } } /** - * A registry to hold all of our enrichment - * configurations. - * - * In the future this may evolve to holding - * all of our enrichments themselves. - * - * @param configs Map whose keys are enrichment - * names and whose values are the - * corresponding enrichment objects + * A registry to hold all of our enrichment configurations. + * In the future this may evolve to holding all of our enrichments themselves. + * @param configs Map whose keys are enrichment names and whose values are the corresponding + * enrichment objects */ -case class EnrichmentRegistry(private val configs: EnrichmentMap) { +final case class EnrichmentRegistry(private val configs: EnrichmentMap) { /** - * A list of all files required by enrichments in the registry. - * This is specified as a pair with the first element providing the - * source location of the file and the second indicating the expected - * local path. + * A list of all files required by enrichments in the registry. This is specified as a pair with + * the first element providing the source location of the file and the second indicating the + * expected local path. */ val filesToCache: List[(URI, String)] = configs.values.flatMap(_.filesToCache).toList /** - * Returns an Option boxing the AnonIpEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the AnonIpEnrichment config value if present, or None if not * @return Option boxing the AnonIpEnrichment instance */ def getAnonIpEnrichment: Option[AnonIpEnrichment] = getEnrichment[AnonIpEnrichment]("anon_ip") /** - * Returns an Option boxing the IpLookupsEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the IpLookupsEnrichment config value if present, or None if not * @return Option boxing the IpLookupsEnrichment instance */ def getIpLookupsEnrichment: Option[IpLookupsEnrichment] = getEnrichment[IpLookupsEnrichment]("ip_lookups") /** - * Returns an Option boxing the RefererParserEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the RefererParserEnrichment config value if present, or None if not * @return Option boxing the RefererParserEnrichment instance */ def getRefererParserEnrichment: Option[RefererParserEnrichment] = getEnrichment[RefererParserEnrichment]("referer_parser") /** - * Returns an Option boxing the CampaignAttributionEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the CampaignAttributionEnrichment config value if present, or None if + * not * @return Option boxing the CampaignAttributionEnrichment instance */ def getCampaignAttributionEnrichment: Option[CampaignAttributionEnrichment] = getEnrichment[CampaignAttributionEnrichment]("campaign_attribution") /** - * Returns an Option boxing the CurrencyConversionEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the CurrencyConversionEnrichment config value if present, or None if + * not * @return Option boxing the CurrencyConversionEnrichment instance */ def getCurrencyConversionEnrichment: Option[CurrencyConversionEnrichment] = getEnrichment[CurrencyConversionEnrichment]("currency_conversion_config") /** - * Returns an Option boxing the UserAgentUtilsEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the UserAgentUtilsEnrichment config value if present, or None if not * @return Option boxing the UserAgentUtilsEnrichment instance */ def getUserAgentUtilsEnrichment: Option[UserAgentUtilsEnrichment.type] = getEnrichment[UserAgentUtilsEnrichment.type]("user_agent_utils_config") /** - * Returns an Option boxing the UaParserEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the UaParserEnrichment config value if present, or None if not * @return Option boxing the UaParserEnrichment instance */ def getUaParserEnrichment: Option[UaParserEnrichment] = @@ -244,90 +218,71 @@ case class EnrichmentRegistry(private val configs: EnrichmentMap) { getEnrichment[YauaaEnrichment]("yauaa_enrichment_config") /** - * Returns an Option boxing the JavascriptScriptEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the JavascriptScriptEnrichment config value if present, or None if not * @return Option boxing the JavascriptScriptEnrichment instance */ def getJavascriptScriptEnrichment: Option[JavascriptScriptEnrichment] = getEnrichment[JavascriptScriptEnrichment]("javascript_script_config") /** - * Returns an Option boxing the getEventFingerprintEnrichment - * config value if present, or None if not - * - * @return Option boxing the getEventFingerprintEnrichment instance + * Returns an Option boxing the EventFingerprintEnrichment config value if present, or None if not + * @return Option boxing the EventFingerprintEnrichment instance */ def getEventFingerprintEnrichment: Option[EventFingerprintEnrichment] = getEnrichment[EventFingerprintEnrichment]("event_fingerprint_config") - /* - * Returns an Option boxing the CookieExtractorEnrichment - * config value if present, or None if not - * + /** + * Returns an Option boxing the CookieExtractorEnrichment config value if present, or None if not * @return Option boxing the CookieExtractorEnrichment instance */ def getCookieExtractorEnrichment: Option[CookieExtractorEnrichment] = getEnrichment[CookieExtractorEnrichment]("cookie_extractor_config") - /* - * Returns an Option boxing the HttpHeaderExtractorEnrichment - * config value if present, or None if not - * + /** + * Returns an Option boxing the HttpHeaderExtractorEnrichment config value if present, or None if + * not * @return Option boxing the HttpHeaderExtractorEnrichment instance */ def getHttpHeaderExtractorEnrichment: Option[HttpHeaderExtractorEnrichment] = getEnrichment[HttpHeaderExtractorEnrichment]("http_header_extractor_config") /** - * Returns an Option boxing the WeatherEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the WeatherEnrichment config value if present, or None if not * @return Option boxing the WeatherEnrichment instance */ def getWeatherEnrichment: Option[WeatherEnrichment] = getEnrichment[WeatherEnrichment]("weather_enrichment_config") /** - * Returns an Option boxing the ApiRequestEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the ApiRequestEnrichment config value if present, or None if not * @return Option boxing the ApiRequestEnrichment instance */ def getApiRequestEnrichment: Option[ApiRequestEnrichment] = getEnrichment[ApiRequestEnrichment]("api_request_enrichment_config") /** - * Returns an Option boxing the SqlQueryEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the SqlQueryEnrichment config value if present, or None if not * @return Option boxing the SqlQueryEnrichment instance */ def getSqlQueryEnrichment: Option[SqlQueryEnrichment] = getEnrichment[SqlQueryEnrichment]("sql_query_enrichment_config") /** - * Returns an Option boxing the PiiPseudonymizerEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the PiiPseudonymizerEnrichment config value if present, or None if not * @return Option boxing the PiiPseudonymizerEnrichment instance */ def getPiiPseudonymizerEnrichment: Option[PiiPseudonymizerEnrichment] = getEnrichment[PiiPseudonymizerEnrichment]("pii_enrichment_config") /** - * Returns an Option boxing the IabEnrichment - * config value if present, or None if not - * + * Returns an Option boxing the IabEnrichment config value if present, or None if not * @return Option boxing the IabEnrichment instance */ def getIabEnrichment: Option[IabEnrichment] = getEnrichment[IabEnrichment]("iab_spiders_and_robots_enrichment") /** - * Returns an Option boxing an Enrichment - * config value if present, or None if not - * + * Returns an Option boxing an Enrichment config value if present, or None if not * @tparam A Expected type of the enrichment to get * @param name The name of the enrichment to get * @return Option boxing the enrichment @@ -336,10 +291,9 @@ case class EnrichmentRegistry(private val configs: EnrichmentMap) { configs.get(name).map(cast[A](_)) /** - * Adapted from http://stackoverflow.com/questions/6686992/scala-asinstanceof-with-parameterized-types - * Used to convert an Enrichment to a - * specific subtype of Enrichment - * + * Adapted from + * http://stackoverflow.com/questions/6686992/scala-asinstanceof-with-parameterized-types + * Used to convert an Enrichment to a specific subtype of Enrichment * @tparam A Type to cast to * @param a The object to cast to type A * @return a, converted to type A diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala index a78d1305a..c6615a9b0 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala @@ -22,30 +22,22 @@ import org.joda.time.format.DateTimeFormat import scalaz._ import Scalaz._ -/** - * Holds the enrichments related to events. - */ +/** Holds the enrichments related to events. */ object EventEnrichments { - /** - * A Redshift-compatible timestamp format - */ - private val TstampFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(DateTimeZone.UTC) + /** A Redshift-compatible timestamp format */ + private val TstampFormat = + DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(DateTimeZone.UTC) /** - * Converts a Joda DateTime into - * a Redshift-compatible timestamp String. - * - * @param datetime The Joda DateTime - * to convert to a timestamp String + * Converts a Joda DateTime into a Redshift-compatible timestamp String. + * @param datetime The Joda DateTime to convert to a timestamp String * @return the timestamp String */ def toTimestamp(datetime: DateTime): String = TstampFormat.print(datetime) /** - * Converts a Redshift-compatible timestamp String - * back into a Joda DateTime. - * + * Converts a Redshift-compatible timestamp String back into a Joda DateTime. * @param timestamp The timestamp String to convert * @return the Joda DateTime */ @@ -53,7 +45,6 @@ object EventEnrichments { /** * Make a collector_tstamp Redshift-compatible - * * @param Optional collectorTstamp * @return Validation boxing the result of making the timestamp Redshift-compatible */ @@ -71,15 +62,12 @@ object EventEnrichments { /** * Calculate the derived timestamp - * * If dvce_sent_tstamp and dvce_created_tstamp are not null and the former is after the latter, * add the difference between the two to the collector_tstamp. * Otherwise just return the collector_tstamp. - * * TODO: given missing collectorTstamp is invalid, consider updating this signature to * `..., collectorTstamp: String): Validation[String, String]` and making the call to this * function in the EnrichmentManager dependent on a Success(collectorTstamp). - * * @param dvceSentTstamp * @param dvceCreatedTstamp * @param collectorTstamp @@ -111,19 +99,10 @@ object EventEnrichments { } /** - * Extracts the timestamp from the - * format as laid out in the Tracker - * Protocol: - * + * Extracts the timestamp from the format as laid out in the Tracker Protocol: * https://github.com/snowplow/snowplow/wiki/snowplow-tracker-protocol#wiki-common-params - * - * @param tstamp The timestamp as - * stored in the Tracker - * Protocol - * @return a Tuple of two Strings - * (date and time), or an - * error message if the - * format was invalid + * @param tstamp The timestamp as stored in the Tracker Protocol + * @return a Tuple of two Strings (date and time), or an error message if the format was invalid */ val extractTimestamp: (String, String) => ValidatedString = (field, tstamp) => try { @@ -140,16 +119,11 @@ object EventEnrichments { } /** - * Turns an event code into a valid event type, - * e.g. "pv" -> "page_view". See the Tracker + * Turns an event code into a valid event type, e.g. "pv" -> "page_view". See the Tracker * Protocol for details: - * * https://github.com/snowplow/snowplow/wiki/snowplow-tracker-protocol#wiki-event2 - * * @param eventCode The event code - * @return the event type, or an error message - * if not recognised, boxed in a Scalaz - * Validation + * @return the event type, or an error message if not recognised, boxed in a Scalaz Validation */ val extractEventType: (String, String) => ValidatedString = (field, code) => code match { @@ -165,13 +139,9 @@ object EventEnrichments { } /** - * Returns a unique event ID. The event ID is - * generated as a type 4 UUID, then converted + * Returns a unique event ID. The event ID is generated as a type 4 UUID, then converted * to a String. - * - * () on the function signature because it's - * not pure - * + * () on the function signature because it's not pure * @return the unique event ID */ def generateEventId(): String = UUID.randomUUID().toString diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala index f376c702c..2264a8224 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala @@ -13,45 +13,30 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments +import io.circe._ +import io.circe.syntax._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import generated.ProjectSettings import utils.{ConversionUtils => CU} -/** - * Miscellaneous enrichments which don't fit into - * one of the other modules. - */ +/** Miscellaneous enrichments which don't fit into one of the other modules. */ object MiscEnrichments { /** - * The version of this ETL. Appends this version - * to the supplied "host" ETL. - * - * @param hostEtlVersion The version of the host ETL - * running this library + * The version of this ETL. Appends this version to the supplied "host" ETL. + * @param hostEtlVersion The version of the host ETL running this library * @return the complete ETL version */ def etlVersion(hostEtlVersion: String): String = "%s-common-%s".format(hostEtlVersion, ProjectSettings.version) /** - * Validate the specified - * platform. - * - * @param field The name of - * the field being - * processed - * @param platform The code - * for the platform - * generating this - * event. - * @return a Scalaz - * ValidatedString. + * Validate the specified platform. + * @param field The name of the field being processed + * @param platform The code for the platform generating this event. + * @return a Scalaz ValidatedString. */ val extractPlatform: (String, String) => ValidatedString = (field, platform) => { platform match { @@ -67,16 +52,12 @@ object MiscEnrichments { } } - /** - * Identity transform. - * Straight passthrough. - */ + /** Identity transform. Straight passthrough. */ val identity: (String, String) => ValidatedString = (field, value) => value.success - /** - * Make a String TSV safe - */ - val toTsvSafe: (String, String) => ValidatedString = (field, value) => CU.makeTsvSafe(value).success + /** Make a String TSV safe */ + val toTsvSafe: (String, String) => ValidatedString = (field, value) => + CU.makeTsvSafe(value).success /** * The X-Forwarded-For header can contain a comma-separated list of IPs especially if it has @@ -91,14 +72,14 @@ object MiscEnrichments { /** * Turn a list of custom contexts into a self-describing JSON - * * @param derivedContexts * @return Self-describing JSON of custom contexts */ - def formatDerivedContexts(derivedContexts: List[JObject]): String = - compact( - render( - ("schema" -> "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1") ~ - ("data" -> JArray(derivedContexts)) - )) + def formatDerivedContexts(derivedContexts: List[Json]): String = + Json + .obj( + "schema" := "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1", + "data" := Json.arr(derivedContexts: _*) + ) + .noSpaces } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala index e61b3998a..310b7ad84 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala @@ -26,15 +26,19 @@ import utils.shredder.Shredder object SchemaEnrichment { private object Schemas { - val pageViewSchema = SchemaKey("com.snowplowanalytics.snowplow", "page_view", "jsonschema", "1-0-0").success - val pagePingSchema = SchemaKey("com.snowplowanalytics.snowplow", "page_ping", "jsonschema", "1-0-0").success - val transactionSchema = SchemaKey("com.snowplowanalytics.snowplow", "transaction", "jsonschema", "1-0-0").success + val pageViewSchema = + SchemaKey("com.snowplowanalytics.snowplow", "page_view", "jsonschema", "1-0-0").success + val pagePingSchema = + SchemaKey("com.snowplowanalytics.snowplow", "page_ping", "jsonschema", "1-0-0").success + val transactionSchema = + SchemaKey("com.snowplowanalytics.snowplow", "transaction", "jsonschema", "1-0-0").success val transactionItemSchema = SchemaKey("com.snowplowanalytics.snowplow", "transaction_item", "jsonschema", "1-0-0").success val structSchema = SchemaKey("com.google.analytics", "event", "jsonschema", "1-0-0").success } - def extractSchema(event: EnrichedEvent)(implicit resolver: Resolver): Validation[String, SchemaKey] = + def extractSchema(event: EnrichedEvent)( + implicit resolver: Resolver): Validation[String, SchemaKey] = event.event match { case "page_view" => Schemas.pageViewSchema case "page_ping" => Schemas.pagePingSchema @@ -45,7 +49,8 @@ object SchemaEnrichment { case eventType => "Unrecognized event [%s]".format(eventType).fail } - private def extractUnstructSchema(event: EnrichedEvent)(implicit resolver: Resolver): Validation[String, SchemaKey] = + private def extractUnstructSchema(event: EnrichedEvent)( + implicit resolver: Resolver): Validation[String, SchemaKey] = Shredder.extractUnstructEvent(event) match { case Some(Success(List(json))) => parseSchemaKey(Option(json.get("schema"))) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala index 59e802196..b04b58519 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala @@ -15,75 +15,54 @@ package enrichments.registry import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ -import org.json4s.{DefaultFormats, JValue} +import io.circe._ import scalaz._ import Scalaz._ -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils import java.net.{Inet4Address, Inet6Address} import com.google.common.net.{InetAddresses => GuavaInetAddress} import scala.util.{Failure, Success, Try} -/** - * Companion object. Lets us create a AnonIpEnrichment - * from a JValue. - */ +/** Companion object. Lets us create a AnonIpEnrichment from a Json. */ object AnonIpEnrichment extends ParseableEnrichment { - implicit val formats = DefaultFormats - - val supportedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "anon_ip", "jsonschema", 1, 0) + val supportedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "anon_ip", "jsonschema", 1, 0) /** * Creates an AnonIpEnrichment instance from a JValue. - * * @param config The anon_ip enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return a configured AnonIpEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[AnonIpEnrichment] = { - - def extractParam(name: String) = - ScalazJson4sUtils.extract[Int](config, "parameters", name) - - isParseable(config, schemaKey).flatMap(_ => { + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[AnonIpEnrichment] = + isParseable(config, schemaKey).flatMap(conf => { (for { - paramIPv4Octet <- extractParam("anonOctets") - paramIPv6Segment <- extractParam("anonSegments").orElse(paramIPv4Octet.success) - ipv4Octets <- AnonIPv4Octets.fromInt(paramIPv4Octet) - ipv6Segment <- AnonIPv6Segments.fromInt(paramIPv6Segment) + paramIPv4Octet <- ScalazCirceUtils.extract[Int](config, "parameters", "anonOctets") + paramIPv6Segment <- ScalazCirceUtils + .extract[Int](config, "parameters", "anonSegments") + .orElse(paramIPv4Octet.success) + ipv4Octets <- AnonIPv4Octets.fromInt(paramIPv4Octet) + ipv6Segment <- AnonIPv6Segments.fromInt(paramIPv6Segment) enrich = AnonIpEnrichment(ipv4Octets, ipv6Segment) } yield enrich).toValidationNel }) - } - } -/** - * How many octets (ipv4) to anonymize? - */ +/** How many octets (ipv4) to anonymize */ object AnonIPv4Octets extends Enumeration { - type AnonIPv4Octets = Value - val One = Value(1, "1") val Two = Value(2, "2") val Three = Value(3, "3") val All = Value(4, "4") /** - * Convert a Stringly-typed integer - * into the corresponding AnonIPv4Octets - * Enum Value. - * - * Update the Validation Error if the - * conversion isn't possible. - * - * @param anonIPv4Octets A String holding - * the number of IP address - * octets to anonymize + * Convert a Stringly-typed integer into the corresponding AnonOctets Enum Value. + * Update the Validation Error if the conversion isn't possible. + * @param anonIPv4Octets A String holding the number of IP address octets to anonymize * @return a Validation-boxed AnonIPv4Octets */ def fromInt(anonIPv4Octets: Int): ValidatedMessage[AnonIPv4Octets] = @@ -102,14 +81,14 @@ object AnonIPv6Segments extends Enumeration { type AnonIPv6Segments = Value - val One = Value(1, "1") - val Two = Value(2, "2") + val One = Value(1, "1") + val Two = Value(2, "2") val Three = Value(3, "3") - val Four = Value(4, "4") - val Five = Value(5, "5") - val Six = Value(6, "6") + val Four = Value(4, "4") + val Five = Value(5, "5") + val Six = Value(6, "6") val Seven = Value(7, "7") - val All = Value(8, "8") + val All = Value(8, "8") /** * Convert a Stringly-typed integer @@ -145,7 +124,7 @@ case class AnonIpEnrichment( ) extends Enrichment { val IPv4MappedAddressPrefix = "::FFFF:" - val MaskChar = "x" + val MaskChar = "x" /** * Anonymize the supplied IP address. @@ -179,7 +158,7 @@ case class AnonIpEnrichment( Option(ipOrNull).map { ip => Try(GuavaInetAddress.forString(ip)) .map { - case _: Inet4Address => anonymizeIpV4(ip) + case _: Inet4Address => anonymizeIpV4(ip) case ipv6: Inet6Address => anonymizeIpV6(ipv6.getHostAddress) } .getOrElse(tryAnonymizingInvalidIp(ip)) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala index 00ce351da..0eee2c781 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala @@ -14,22 +14,18 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} -import org.json4s.{DefaultFormats, JValue} +import io.circe._ import scalaz._ import Scalaz._ import utils.MapTransformer.SourceMap -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Companion object. Lets us create a - * CampaignAttributionEnrichment from a JValue - */ +/** Companion object. Lets us create a CampaignAttributionEnrichment from a Json */ object CampaignAttributionEnrichment extends ParseableEnrichment { - implicit val formats = DefaultFormats - - val supportedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "campaign_attribution", "jsonschema", 1, 0) + val supportedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "campaign_attribution", "jsonschema", 1, 0) val DefaultNetworkMap = Map( "gclid" -> "Google", @@ -39,22 +35,27 @@ object CampaignAttributionEnrichment extends ParseableEnrichment { /** * Creates a CampaignAttributionEnrichment instance from a JValue. - * * @param config The referer_parser enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return a configured CampaignAttributionEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[CampaignAttributionEnrichment] = + def parse( + config: Json, + schemaKey: SchemaKey + ): ValidatedNelMessage[CampaignAttributionEnrichment] = isParseable(config, schemaKey).flatMap(conf => { (for { - medium <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "fields", "mktMedium") - source <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "fields", "mktSource") - term <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "fields", "mktTerm") - content <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "fields", "mktContent") - campaign <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "fields", "mktCampaign") - - customClickMap = ScalazJson4sUtils + medium <- ScalazCirceUtils + .extract[List[String]](config, "parameters", "fields", "mktMedium") + source <- ScalazCirceUtils + .extract[List[String]](config, "parameters", "fields", "mktSource") + term <- ScalazCirceUtils.extract[List[String]](config, "parameters", "fields", "mktTerm") + content <- ScalazCirceUtils + .extract[List[String]](config, "parameters", "fields", "mktContent") + campaign <- ScalazCirceUtils + .extract[List[String]](config, "parameters", "fields", "mktCampaign") + + customClickMap = ScalazCirceUtils .extract[Map[String, String]](config, "parameters", "fields", "mktClickId") .fold( // Assign empty Map on missing property for backwards compatibility with schema version 1-0-0 @@ -76,7 +77,6 @@ object CampaignAttributionEnrichment extends ParseableEnrichment { /** * Class for a marketing campaign - * * @param medium Campaign medium * @param source Campaign source * @param term Campaign term @@ -85,7 +85,7 @@ object CampaignAttributionEnrichment extends ParseableEnrichment { * @param clickId Click ID * @param network Advertising network */ -case class MarketingCampaign( +final case class MarketingCampaign( medium: Option[String], source: Option[String], term: Option[String], @@ -97,7 +97,6 @@ case class MarketingCampaign( /** * Config for a campaign_attribution enrichment - * * @param mediumParameters List of marketing medium parameters * @param sourceParameters List of marketing source parameters * @param termParameters List of marketing term parameters @@ -105,7 +104,7 @@ case class MarketingCampaign( * @param campaignParameters List of marketing campaign parameters * @param mktClick: Map of click ID parameters to networks */ -case class CampaignAttributionEnrichment( +final case class CampaignAttributionEnrichment( mediumParameters: List[String], sourceParameters: List[String], termParameters: List[String], @@ -115,11 +114,9 @@ case class CampaignAttributionEnrichment( ) extends Enrichment { /** - * Find the first string in parameterList which is a key of - * sourceMap and return the value of that key in sourceMap. - * - * @param parameterList List of accepted campaign parameter - * names in order of decreasing precedence + * Find the first string in parameterList which is a key of sourceMap and return the value of that + * key in sourceMap. + * @param parameterList List of accepted parameter names in order of decreasing precedence * @param sourceMap Map of key-value pairs in URI querystring * @return Option boxing the value of the campaign parameter */ @@ -128,13 +125,8 @@ case class CampaignAttributionEnrichment( /** * Extract the marketing fields from a URL. - * - * @param nvPairs The querystring to extract - * marketing fields from - * @return the MarketingCampaign - * or an error message, - * boxed in a Scalaz - * Validation + * @param nvPairs The querystring to extract marketing fields from + * @return the MarketingCampaign or an error message, boxed in a Scalaz Validation */ def extractMarketingFields(nvPairs: SourceMap): ValidationNel[String, MarketingCampaign] = { val medium = getFirstParameter(mediumParameters, nvPairs) @@ -145,7 +137,9 @@ case class CampaignAttributionEnrichment( val (clickId, network) = Unzip[Option].unzip( - clickIdParameters.find(pair => nvPairs.contains(pair._1)).map(pair => (nvPairs(pair._1), pair._2))) + clickIdParameters + .find(pair => nvPairs.contains(pair._1)) + .map(pair => (nvPairs(pair._1), pair._2))) MarketingCampaign(medium, source, term, content, campaign, clickId, network).success.toValidationNel } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala index 5767ddd19..1a16453dc 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala @@ -14,32 +14,29 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} +import io.circe._ +import io.circe.syntax._ import org.apache.http.message.BasicHeaderValueParser import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils object CookieExtractorEnrichmentConfig extends ParseableEnrichment { - implicit val formats = DefaultFormats - - val supportedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "cookie_extractor_config", "jsonschema", 1, 0) + val supportedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "cookie_extractor_config", "jsonschema", 1, 0) /** * Creates a CookieExtractorEnrichment instance from a JValue. - * * @param config The cookie_extractor enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return a configured CookieExtractorEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[CookieExtractorEnrichment] = + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[CookieExtractorEnrichment] = isParseable(config, schemaKey).flatMap(conf => { (for { - cookieNames <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "cookies") + cookieNames <- ScalazCirceUtils.extract[List[String]](config, "parameters", "cookies") enrich = CookieExtractorEnrichment(cookieNames) } yield enrich).toValidationNel }) @@ -47,20 +44,17 @@ object CookieExtractorEnrichmentConfig extends ParseableEnrichment { /** * Enrichment extracting certain cookies from headers. - * * @param cookieNames Names of the cookies to be extracted */ -case class CookieExtractorEnrichment( - cookieNames: List[String] -) extends Enrichment { +final case class CookieExtractorEnrichment(cookieNames: List[String]) extends Enrichment { - def extract(headers: List[String]): List[JsonAST.JObject] = { + def extract(headers: List[String]): List[Json] = { // rfc6265 - sections 4.2.1 and 4.2.2 - val cookies = headers.flatMap { header => header.split(":", 2) match { case Array("Cookie", value) => - val nameValuePairs = BasicHeaderValueParser.parseParameters(value, BasicHeaderValueParser.INSTANCE) + val nameValuePairs = + BasicHeaderValueParser.parseParameters(value, BasicHeaderValueParser.INSTANCE) val filtered = nameValuePairs.filter { nvp => cookieNames.contains(nvp.getName) @@ -72,10 +66,16 @@ case class CookieExtractorEnrichment( }.flatten cookies.map { cookie => - (("schema" -> "iglu:org.ietf/http_cookie/jsonschema/1-0-0") ~ - ("data" -> - ("name" -> cookie.getName) ~ - ("value" -> cookie.getValue))) + Json.obj( + "schema" := "iglu:org.ietf/http_cookie/jsonschema/1-0-0", + "data" := Json.obj( + "name" := stringToJson(cookie.getName), + "value" := stringToJson(cookie.getValue) + ) + ) } } + + private def stringToJson(str: String): Json = + Option(str).map(Json.fromString).getOrElse(Json.Null) } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala index a9d468e21..02f915a75 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala @@ -21,77 +21,82 @@ import com.snowplowanalytics.forex.oerclient._ import com.snowplowanalytics.forex.{Forex, ForexConfig} import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ import org.joda.time.DateTime -import org.json4s.{DefaultFormats, JValue} import scalaz._ import Scalaz._ -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Companion object. Lets us create an CurrencyConversionEnrichment - * instance from a JValue. - */ +/** Companion object. Lets us create an CurrencyConversionEnrichment instance from a Json. */ object CurrencyConversionEnrichmentConfig extends ParseableEnrichment { - implicit val formats = DefaultFormats - val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow", "currency_conversion_config", "jsonschema", 1, 0) + SchemaCriterion( + "com.snowplowanalytics.snowplow", + "currency_conversion_config", + "jsonschema", + 1, + 0 + ) // Creates a CurrencyConversionEnrichment instance from a JValue - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[CurrencyConversionEnrichment] = - isParseable(config, schemaKey).flatMap(conf => { + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[CurrencyConversionEnrichment] = + isParseable(config, schemaKey).flatMap { conf => (for { - apiKey <- ScalazJson4sUtils.extract[String](config, "parameters", "apiKey") - baseCurrency <- ScalazJson4sUtils.extract[String](config, "parameters", "baseCurrency") - accountType <- (ScalazJson4sUtils.extract[String](config, "parameters", "accountType") match { - case Success("DEVELOPER") => DeveloperAccount.success - case Success("ENTERPRISE") => EnterpriseAccount.success - case Success("UNLIMITED") => UnlimitedAccount.success - - // Should never happen (prevented by schema validation) - case Success(s) => - "accountType [%s] is not one of DEVELOPER, ENTERPRISE, and UNLIMITED".format(s).toProcessingMessage.fail - case Failure(f) => Failure(f) - }) - rateAt <- ScalazJson4sUtils.extract[String](config, "parameters", "rateAt") + apiKey <- ScalazCirceUtils.extract[String](config, "parameters", "apiKey") + baseCurrency <- ScalazCirceUtils.extract[String](config, "parameters", "baseCurrency") + accountType <- ScalazCirceUtils + .extract[String](config, "parameters", "accountType") + .flatMap { + case "DEVELOPER" => DeveloperAccount.success + case "ENTERPRISE" => EnterpriseAccount.success + case "UNLIMITED" => UnlimitedAccount.success + // Should never happen (prevented by schema validation) + case s => + "accountType [%s] is not one of DEVELOPER, ENTERPRISE, and UNLIMITED" + .format(s) + .toProcessingMessage + .fail + } + rateAt <- ScalazCirceUtils.extract[String](config, "parameters", "rateAt") enrich = CurrencyConversionEnrichment(accountType, apiKey, baseCurrency, rateAt) } yield enrich).toValidationNel - }) + } } /** * Configuration for a currency_conversion enrichment - * * @param apiKey OER authentication * @param baseCurrency Currency to which to convert * @param rateAt Which exchange rate to use - "EOD_PRIOR" for "end of previous day". */ -case class CurrencyConversionEnrichment(accountType: AccountType, apiKey: String, baseCurrency: String, rateAt: String) - extends Enrichment { - +final case class CurrencyConversionEnrichment( + accountType: AccountType, + apiKey: String, + baseCurrency: String, + rateAt: String +) extends Enrichment { val fx = Forex(ForexConfig(), OerClientConfig(apiKey, accountType)) /** * Attempt to convert if the initial currency and value are both defined - * * @param inputCurrency Option boxing the initial currency if it is present * @param value Option boxing the amount to convert * @return None.success if the inputs were not both defined, - * otherwise Validation[Option[_]] boxing the result of the conversion + * otherwise Validation[Option[_]] boxing the result of the conversion */ private def performConversion( initialCurrency: Option[String], value: Option[Double], - tstamp: DateTime): Validation[String, Option[String]] = + tstamp: DateTime + ): Validation[String, Option[String]] = (initialCurrency, value) match { case (Some(ic), Some(v)) => fx.convert(v, ic).to(baseCurrency).at(tstamp) match { - case Left(l) => { + case Left(l) => val errorType = l.errorType.getClass.getSimpleName.replace("$", "") s"Open Exchange Rates error, type: [$errorType], message: [${l.errorMessage}]".failure - } case Right(s) => (s.getAmount().toPlainString()).some.success } case _ => None.success @@ -99,7 +104,6 @@ case class CurrencyConversionEnrichment(accountType: AccountType, apiKey: String /** * Converts currency using Scala Forex - * * @param trCurrency Initial transaction currency * @param trTotal Total transaction value * @param trTax Transaction tax @@ -116,8 +120,8 @@ case class CurrencyConversionEnrichment(accountType: AccountType, apiKey: String trShipping: Option[Double], tiCurrency: Option[String], tiPrice: Option[Double], - collectorTstamp: Option[DateTime]) - : ValidationNel[String, (Option[String], Option[String], Option[String], Option[String])] = + collectorTstamp: Option[DateTime] + ): ValidationNel[String, (Option[String], Option[String], Option[String], Option[String])] = collectorTstamp match { case Some(tstamp) => try { @@ -129,8 +133,10 @@ case class CurrencyConversionEnrichment(accountType: AccountType, apiKey: String (_, _, _, _) } } catch { - case e: NoSuchElementException => "Base currency [%s] not supported: [%s]".format(baseCurrency, e).failNel - case f: UnknownHostException => "Could not connect to Open Exchange Rates: [%s]".format(f).failNel + case e: NoSuchElementException => + "Base currency [%s] not supported: [%s]".format(baseCurrency, e).failNel + case f: UnknownHostException => + "Could not connect to Open Exchange Rates: [%s]".format(f).failNel case NonFatal(g) => "Unexpected exception converting currency: [%s]".format(g).failNel } case None => "Collector timestamp missing".failNel // This should never happen diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala index 960dcca08..964132dc9 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala @@ -15,77 +15,76 @@ package enrichments.registry import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ import org.apache.commons.codec.digest.DigestUtils import scalaz._ import Scalaz._ -import org.json4s._ -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Lets us create an EventFingerprintEnrichmentConfig from a JValue. - */ +/** Lets us create an EventFingerprintEnrichmentConfig from a Json. */ object EventFingerprintEnrichmentConfig extends ParseableEnrichment { - implicit val formats = DefaultFormats - val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow", "event_fingerprint_config", "jsonschema", 1, 0) + SchemaCriterion( + "com.snowplowanalytics.snowplow", + "event_fingerprint_config", + "jsonschema", + 1, + 0 + ) /** * Creates an EventFingerprintEnrichment instance from a JValue. - * * @param config The enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported fby this enrichment * @return a configured EventFingerprintEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[EventFingerprintEnrichment] = + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[EventFingerprintEnrichment] = isParseable(config, schemaKey).flatMap(conf => { (for { - excludedParameters <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "excludeParameters") - algorithmName <- ScalazJson4sUtils.extract[String](config, "parameters", "hashAlgorithm") + excludedParameters <- ScalazCirceUtils + .extract[List[String]](config, "parameters", "excludeParameters") + algorithmName <- ScalazCirceUtils.extract[String](config, "parameters", "hashAlgorithm") algorithm <- getAlgorithm(algorithmName) } yield EventFingerprintEnrichment(algorithm, excludedParameters)).toValidationNel }) /** * Look up the fingerprinting algorithm by name - * * @param algorithmName * @return A hashing algorithm */ - private[registry] def getAlgorithm(algorithmName: String): ValidatedMessage[String => String] = algorithmName match { - case "MD5" => ((s: String) => DigestUtils.md5Hex(s)).success - case "SHA1" => ((s: String) => DigestUtils.sha1Hex(s)).success - case "SHA256" => ((s: String) => DigestUtils.sha256Hex(s)).success - case "SHA384" => ((s: String) => DigestUtils.sha384Hex(s)).success - case "SHA512" => ((s: String) => DigestUtils.sha512Hex(s)).success - case other => s"[$other] is not a supported event fingerprint generation algorithm".toProcessingMessage.fail - } + private[registry] def getAlgorithm(algorithmName: String): ValidatedMessage[String => String] = + algorithmName match { + case "MD5" => ((s: String) => DigestUtils.md5Hex(s)).success + case "SHA1" => ((s: String) => DigestUtils.sha1Hex(s)).success + case "SHA256" => ((s: String) => DigestUtils.sha256Hex(s)).success + case "SHA384" => ((s: String) => DigestUtils.sha384Hex(s)).success + case "SHA512" => ((s: String) => DigestUtils.sha512Hex(s)).success + case other => + s"[$other] is not a supported event fingerprint generation algorithm".toProcessingMessage.fail + } } -/** - * Companion object - */ object EventFingerprintEnrichment { private val UnitSeparator = "\u001f" } /** * Config for an event fingerprint enrichment - * * @param algorithm Hashing algorithm * @param excludedParameters List of querystring parameters to exclude from the calculation * @return Event fingerprint */ -case class EventFingerprintEnrichment(algorithm: String => String, excludedParameters: List[String]) - extends Enrichment { +final case class EventFingerprintEnrichment( + algorithm: String => String, + excludedParameters: List[String] +) extends Enrichment { /** * Calculate an event fingerprint using all querystring fields except the excludedParameters - * * @param parameterMap * @return Event fingerprint */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala index 4283e629b..dbd6ed408 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala @@ -14,30 +14,34 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} -import org.json4s._ -import org.json4s.JsonDSL._ +import io.circe._ +import io.circe.syntax._ -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils object HttpHeaderExtractorEnrichmentConfig extends ParseableEnrichment { - implicit val formats = DefaultFormats - val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow.enrichments", "http_header_extractor_config", "jsonschema", 1, 0) + SchemaCriterion( + "com.snowplowanalytics.snowplow.enrichments", + "http_header_extractor_config", + "jsonschema", + 1, + 0) /** - * Creates a HttpHeaderExtractorEnrichment instance from a JValue. - * + * Creates a HttpHeaderExtractorEnrichment instance from a Json. * @param config The header_extractor enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return a configured HeaderExtractorEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[HttpHeaderExtractorEnrichment] = + def parse( + config: Json, + schemaKey: SchemaKey + ): ValidatedNelMessage[HttpHeaderExtractorEnrichment] = isParseable(config, schemaKey).flatMap(conf => { (for { - headersPattern <- ScalazJson4sUtils.extract[String](config, "parameters", "headersPattern") + headersPattern <- ScalazCirceUtils.extract[String](config, "parameters", "headersPattern") enrich = HttpHeaderExtractorEnrichment(headersPattern) } yield enrich).toValidationNel }) @@ -45,14 +49,12 @@ object HttpHeaderExtractorEnrichmentConfig extends ParseableEnrichment { /** * Enrichment extracting certain headers from headers. - * * @param headersPattern Names of the headers to be extracted */ -case class HttpHeaderExtractorEnrichment(headersPattern: String) extends Enrichment { - +final case class HttpHeaderExtractorEnrichment(headersPattern: String) extends Enrichment { case class Header(name: String, value: String) - def extract(headers: List[String]): List[JsonAST.JObject] = { + def extract(headers: List[String]): List[Json] = { val httpHeaders = headers.flatMap { header => header.split(":", 2) match { case Array(name, value) if name.matches(headersPattern) => @@ -62,10 +64,13 @@ case class HttpHeaderExtractorEnrichment(headersPattern: String) extends Enrichm } httpHeaders.map { header => - (("schema" -> "iglu:org.ietf/http_header/jsonschema/1-0-0") ~ - ("data" -> - ("name" -> header.name.trim) ~ - ("value" -> header.value.trim))) + Json.obj( + "schema" := Json.fromString("iglu:org.ietf/http_header/jsonschema/1-0-0"), + "data" := Json.obj( + "name" := Json.fromString(header.name.trim), + "value" := Json.fromString(header.value.trim) + ) + ) } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala index d29996076..578ce6443 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala @@ -16,25 +16,21 @@ package enrichments.registry import java.io.File import java.net.{InetAddress, URI, UnknownHostException} -import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} +import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ import com.snowplowanalytics.iab.spidersandrobotsclient.IabClient +import io.circe._ +import io.circe.generic.auto._ +import io.circe.syntax._ import org.joda.time.DateTime -import org.json4s.{DefaultFormats, Extraction, JObject, JValue} -import org.json4s.JsonDSL._ import scalaz._ import Scalaz._ -import utils.{ConversionUtils, ScalazJson4sUtils} +import utils.{ConversionUtils, ScalazCirceUtils} -/** - * Companion object. Lets us create an IabEnrichment - * instance from a JValue. - */ +/** Companion object. Lets us create an IabEnrichment instance from a Json. */ object IabEnrichment extends ParseableEnrichment { - implicit val formats = DefaultFormats - val supportedSchema = SchemaCriterion( "com.snowplowanalytics.snowplow.enrichments", "iab_spiders_and_robots_enrichment", @@ -43,86 +39,65 @@ object IabEnrichment extends ParseableEnrichment { 0) /** - * Creates an IabEnrichment instance from a JValue. - * + * Creates an IabEnrichment instance from a Json. * @param config The iab_spiders_and_robots_enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment - * @param localMode Whether to use the local IAB database file - * Enabled for tests + * @param schemaKey provided for the enrichment, must be supported by this enrichment + * @param localMode Whether to use the local IAB database file, enabled for tests * @return a configured IabEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey, localMode: Boolean): ValidatedNelMessage[IabEnrichment] = + def parse( + config: Json, + schemaKey: SchemaKey, + localMode: Boolean + ): ValidatedNelMessage[IabEnrichment] = isParseable(config, schemaKey).flatMap { conf => - def uri(name: String) = getIabDbFromName(conf, name).sequenceU - - def string(name: String) = getStringFromName(conf, name).sequenceU - + def uri(name: String) = getIabDbFromName(conf, name) (uri("ipFile") |@| uri("excludeUseragentFile") |@| uri("includeUseragentFile")) { - IabEnrichment(_, _, _, localMode) + case (ip, exclude, include) => IabEnrichment(ip.some, exclude.some, include.some, localMode) } } /** * Creates IabDatabase instances used in the IabEnrichment case class. - * - * @param config The iab_spiders_and_robots_enrichment JSON - * @param name The name of the field. - * e.g. "ipFile", "excluseUseragentFile", "includeUseragentFile" - * @return None if the field does not exist, - * Some(Failure) if the URI is invalid, - * Some(Success) if it is found - */ - private def getIabDbFromName(config: JValue, name: String): Option[ValidatedNelMessage[IabDatabase]] = - if (ScalazJson4sUtils.fieldExists(config, "parameters", name)) { - val uri = ScalazJson4sUtils.extract[String](config, "parameters", name, "uri") - val db = ScalazJson4sUtils.extract[String](config, "parameters", name, "database") - - (uri.toValidationNel |@| db.toValidationNel) { (uri, db) => - getDatabaseUri(uri, db).toValidationNel.map[IabDatabase](u => IabDatabase(name, u, db)) - }.flatMap(identity).some - - } else None - - /** - * Extracts simple string fields from an enrichment JSON. - * * @param config The iab_spiders_and_robots_enrichment JSON - * @param name The name of the field. - * e.g. "ipFile", "excluseUseragentFile", "includeUseragentFile" - * @return None if the field does not exist, - * Some(Success) if it is found + * @param name of the field, e.g. "ipFile", "excluseUseragentFile", "includeUseragentFile" + * @return None if the field does not exist, Some(Failure) if the URI is invalid, Some(Success) if + * it is found */ - private def getStringFromName(config: JValue, name: String): Option[ValidatedNelMessage[String]] = - if (ScalazJson4sUtils.fieldExists(config, "parameters", name)) { - ScalazJson4sUtils.extract[String](config, "parameters", name).toValidationNel.some - } else None + private def getIabDbFromName( + config: Json, + name: String + ): ValidatedNelMessage[IabDatabase] = { + val uri = ScalazCirceUtils.extract[String](config, "parameters", name, "uri") + val db = ScalazCirceUtils.extract[String](config, "parameters", name, "database") + + (uri.toValidationNel |@| db.toValidationNel) { (uri, db) => + getDatabaseUri(uri, db).toValidationNel.map[IabDatabase](u => IabDatabase(name, u, db)) + }.flatMap(identity) + } /** - * Convert the path to the IAB file from a - * String to a Validation[URI]. - * - * @param uri URI to the IAB database file + * Convert the path to the IAB file from a String to a Validation[URI]. + * @param uri URI to the IAB database file * @param database Name of the IAB database * @return a Validation-boxed URI */ private def getDatabaseUri(uri: String, database: String): ValidatedMessage[URI] = ConversionUtils .stringToUri(uri + (if (uri.endsWith("/")) "" else "/") + database) - .flatMap(_ match { + .flatMap { case Some(u) => u.success case None => "URI to IAB file must be provided".fail - }) + } .toProcessingMessage } /** * Contains enrichments based on IAB Spiders&Robots lookup. - * - * @param ipFile (Full URI to the IAB excluded IP list, database name) - * @param excludeUaFile (Full URI to the IAB excluded user agent list, database name) - * @param includeUaFile (Full URI to the IAB included user agent list, database name) - * @param localMode Whether to use the local database file. Enabled for tests. + * @param ipFile (Full URI to the IAB excluded IP list, database name) + * @param excludeUaFile (Full URI to the IAB excluded user agent list, database name) + * @param includeUaFile (Full URI to the IAB included user agent list, database name) + * @param localMode Whether to use the local database file. Enabled for tests. */ case class IabEnrichment( ipFile: Option[IabDatabase], @@ -130,15 +105,12 @@ case class IabEnrichment( includeUaFile: Option[IabDatabase], localMode: Boolean ) extends Enrichment { - private type DbEntry = Option[(Option[URI], String)] private val schemaUri = "iglu:com.iab.snowplow/spiders_and_robots/jsonschema/1-0-0" - private implicit val formats = DefaultFormats // Construct a Tuple3 of all IAB files private val dbs: (DbEntry, DbEntry, DbEntry) = { - def db(iabDb: Option[IabDatabase]): DbEntry = iabDb.map { case IabDatabase(name, uri, db) => if (localMode) { @@ -160,24 +132,23 @@ case class IabEnrichment( // Create an IAB client based on the IAB files list private lazy val iabClient = { def file(db: DbEntry): File = new File(db.get._2) - new IabClient(file(dbs._1), file(dbs._2), file(dbs._3)) } /** - * Get the IAB response containing information about whether an event is a - * spider or robot using the IAB client library. - * - * @param userAgent User agent used to perform the check - * @param ipAddress IP address used to perform the check - * @param accurateAt Date of the event, used to determine whether entries in the - * IAB list are relevant or outdated + * Get the IAB response containing information about whether an event is a spider or robot using + * the IAB client library. + * @param userAgent User agent used to perform the check + * @param ipAddress IP address used to perform the check + * @param accurateAt Date of the event, used to determine whether entries in the IAB list are + * relevant or outdated * @return an IabResponse object */ private[enrichments] def performCheck( userAgent: String, ipAddress: String, - accurateAt: DateTime): Validation[String, IabEnrichmentResponse] = + accurateAt: DateTime + ): Validation[String, IabEnrichmentResponse] = try { val result = iabClient.checkAt(userAgent, InetAddress.getByName(ipAddress), accurateAt.toDate) IabEnrichmentResponse( @@ -191,64 +162,55 @@ case class IabEnrichment( /** * Get the IAB response as a JSON context for a specific event - * - * @param userAgent enriched event optional user agent - * @param ipAddress enriched event optional IP address + * @param userAgent enriched event optional user agent + * @param ipAddress enriched event optional IP address * @param accurateAt enriched event optional datetime * @return IAB response as a self-describing JSON object */ def getIabContext( userAgent: Option[String], ipAddress: Option[String], - accurateAt: Option[DateTime]): Validation[String, JObject] = - getIab(userAgent, ipAddress, accurateAt).map(addSchema) + accurateAt: Option[DateTime] + ): Validation[String, Json] = getIab(userAgent, ipAddress, accurateAt).map(addSchema) /** * Get IAB check response received from the client library and extracted as a JSON object - * * @param userAgent enriched event optional user agent * @param ipAddress enriched event optional IP address - * @param time enriched event optional datetime + * @param time enriched event optional datetime * @return IAB response as JSON object */ private def getIab( userAgent: Option[String], ipAddress: Option[String], - time: Option[DateTime]): Validation[String, JObject] = + time: Option[DateTime] + ): Validation[String, Json] = (userAgent, ipAddress, time) match { - case (Some(ua), Some(ip), Some(t)) => - performCheck(ua, ip, t) match { - case Success(response) => - Extraction.decompose(response) match { - case obj: JObject => obj.success - case _ => s"Couldn't transform IAB response $response into JSON".failure - } - case Failure(message) => message.failure - } + case (Some(ua), Some(ip), Some(t)) => performCheck(ua, ip, t).map(_.asJson) case _ => - s"One of required event fields missing. user agent: $userAgent, ip address: $ipAddress, time: $time".failure + ("One of required event fields missing. " + + s"user agent: $userAgent, ip address: $ipAddress, time: $time").failure } /** * Add Iglu URI to JSON Object - * * @param context IAB context as JSON Object * @return JSON Object wrapped as Self-describing JSON */ - private def addSchema(context: JObject): JObject = - ("schema", schemaUri) ~ (("data", context)) + private def addSchema(context: Json): Json = + Json.obj( + "schema" := schemaUri, + "data" := context + ) } -/** - * Case class copy of `com.snowplowanalytics.iab.spidersandrobotsclient.IabResponse` - */ -private[enrichments] case class IabEnrichmentResponse( +/** Case class copy of `com.snowplowanalytics.iab.spidersandrobotsclient.IabResponse` */ +private[enrichments] final case class IabEnrichmentResponse( spiderOrRobot: Boolean, category: String, reason: String, - primaryImpact: String) + primaryImpact: String +) -/** - * Case class representing an IAB database location - */ -case class IabDatabase(name: String, uri: URI, db: String) +/** Case class representing an IAB database location */ +final case class IabDatabase(name: String, uri: URI, db: String) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala index 67f173bb1..2cda06b8a 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala @@ -19,90 +19,78 @@ import com.snowplowanalytics.maxmind.iplookups.IpLookups import com.snowplowanalytics.maxmind.iplookups.model.IpLookupResult import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ -import org.json4s.{DefaultFormats, JValue} +import io.circe._ import scalaz._ import Scalaz._ -import utils.{ConversionUtils, ScalazJson4sUtils} +import utils.{ConversionUtils, ScalazCirceUtils} -/** - * Companion object. Lets us create an IpLookupsEnrichment - * instance from a JValue. - */ +/** Companion object. Lets us create an IpLookupsEnrichment instance from a Json. */ object IpLookupsEnrichment extends ParseableEnrichment { - - implicit val formats = DefaultFormats - val supportedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "ip_lookups", "jsonschema", 2, 0) + val supportedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "ip_lookups", "jsonschema", 2, 0) /** * Creates an IpLookupsEnrichment instance from a JValue. - * * @param config The ip_lookups enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment - * @param localMode Whether to use the local MaxMind data file - * Enabled for tests + * @param schemaKey provided for the enrichment, must be supported by this enrichment + * @param localMode Whether to use the local MaxMind data file, enabled for tests * @return a configured IpLookupsEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey, localMode: Boolean): ValidatedNelMessage[IpLookupsEnrichment] = - isParseable(config, schemaKey).flatMap(conf => { + def parse( + config: Json, + schemaKey: SchemaKey, + localMode: Boolean + ): ValidatedNelMessage[IpLookupsEnrichment] = + isParseable(config, schemaKey).flatMap { conf => def db(name: String) = getArgumentFromName(conf, name).sequenceU (db("geo") |@| db("isp") |@| db("domain") |@| db("connectionType")) { IpLookupsEnrichment(_, _, _, _, localMode) } - }) + } /** - * Creates the (URI, String) tuple arguments - * which are the case class parameters - * + * Creates the (URI, String) tuple arguments which are the case class parameters * @param conf The ip_lookups enrichment JSON - * @param name The name of the lookup: - * "geo", "isp", "organization", "domain" - * @return None if the database isn't being used, - * Some(Failure) if its URI is invalid, - * Some(Success) if it is found + * @param name The name of the lookup: "geo", "isp", "organization", "domain" + * @return None if the database isn't being used, Some(Failure) if its URI is invalid, + * Some(Success) if it is found */ - private def getArgumentFromName(conf: JValue, name: String): Option[ValidatedNelMessage[(String, URI, String)]] = - if (ScalazJson4sUtils.fieldExists(conf, "parameters", name)) { - val uri = ScalazJson4sUtils.extract[String](conf, "parameters", name, "uri") - val db = ScalazJson4sUtils.extract[String](conf, "parameters", name, "database") + private def getArgumentFromName( + conf: Json, + name: String + ): Option[ValidatedNelMessage[(String, URI, String)]] = + if (conf.hcursor.downField("parameters").downField(name).focus.isDefined) { + val uri = ScalazCirceUtils.extract[String](conf, "parameters", name, "uri") + val db = ScalazCirceUtils.extract[String](conf, "parameters", name, "database") (uri.toValidationNel |@| db.toValidationNel) { (uri, db) => for { u <- (getMaxmindUri(uri, db).toValidationNel: ValidatedNelMessage[URI]) } yield (name, u, db) - - }.flatMap(x => x).some - + }.flatMap(identity).some } else None /** - * Convert the Maxmind file from a - * String to a Validation[URI]. - * - * @param maxmindFile A String holding the - * URI to the hosted MaxMind file - * @param database Name of the MaxMind - * database + * Convert the Maxmind file from a String to a Validation[URI]. + * @param maxmindFile A String holding the URI to the hosted MaxMind file + * @param database Name of the MaxMind database * @return a Validation-boxed URI */ private def getMaxmindUri(uri: String, database: String): ValidatedMessage[URI] = ConversionUtils .stringToUri(uri + "/" + database) - .flatMap(_ match { + .flatMap { case Some(u) => u.success case None => "URI to MaxMind file must be provided".fail - }) + } .toProcessingMessage } /** * Contains enrichments based on IP address. - * * @param uri Full URI to the MaxMind data file * @param database Name of the MaxMind database - * @param geoTuple (Full URI to the geo lookup MaxMind data file, database name) * @param ispTuple (Full URI to the ISP lookup MaxMind data file, database name) * @param domainTuple (Full URI to the domain lookup MaxMind data file, database name) @@ -116,13 +104,11 @@ case class IpLookupsEnrichment( connectionTypeTuple: Option[(String, URI, String)], localMode: Boolean ) extends Enrichment { - private type FinalPath = String private type DbEntry = Option[(Option[URI], FinalPath)] // Construct a Tuple4 of all the IP Lookup databases private val dbs: Tuple4[DbEntry, DbEntry, DbEntry, DbEntry] = { - def db(dbPath: Option[(String, URI, String)]): DbEntry = dbPath.map { case (name, uri, file) => if (localMode) { @@ -131,7 +117,6 @@ case class IpLookupsEnrichment( (Some(uri), "./ip_" + name) } } - (db(geoTuple), db(ispTuple), db(domainTuple), db(connectionTypeTuple)) } @@ -145,19 +130,24 @@ case class IpLookupsEnrichment( // the Dist Cache on HDFS yet private lazy val ipLookups = { def path(db: DbEntry): Option[FinalPath] = db.map(_._2) - IpLookups(path(dbs._1), path(dbs._2), path(dbs._3), path(dbs._4), memCache = true, lruCache = 20000) + IpLookups( + path(dbs._1), + path(dbs._2), + path(dbs._3), + path(dbs._4), + memCache = true, + lruCache = 20000) } /** * Extract the geo-location using the client IP address. * If the IPv4 contains a port, it is removed before performing the lookup. - * * @param geo The IpGeo lookup engine we will use to lookup the client's IP address * @param ip The client's IP address to use to lookup the client's geo-location * @return an IpLookupResult */ def extractIpInformation(ip: String): IpLookupResult = ip match { case EnrichmentManager.IPv4Regex(ipv4WithoutPort) => ipLookups.performLookups(ipv4WithoutPort) - case _ => ipLookups.performLookups(ip) + case _ => ipLookups.performLookups(ip) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala index 46505adce..802677d36 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala @@ -18,37 +18,38 @@ import scala.util.control.NonFatal import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ import org.mozilla.javascript._ +import io.circe._ +import io.circe.parser._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods import outputs.EnrichedEvent -import utils.{ConversionUtils, ScalazJson4sUtils} +import utils.{ConversionUtils, ScalazCirceUtils} -/** - * Lets us create a JavascriptScriptEnrichment from a JValue. - */ +/** Lets us create a JavascriptScriptEnrichment from a Json. */ object JavascriptScriptEnrichmentConfig extends ParseableEnrichment { - - implicit val formats = DefaultFormats - val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow", "javascript_script_config", "jsonschema", 1, 0) + SchemaCriterion( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + 1, + 0 + ) /** * Creates a JavascriptScriptEnrichment instance from a JValue. - * * @param config The JavaScript script enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return a configured JavascriptScriptEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[JavascriptScriptEnrichment] = + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[JavascriptScriptEnrichment] = isParseable(config, schemaKey).flatMap(conf => { (for { - encoded <- ScalazJson4sUtils.extract[String](config, "parameters", "script") - raw <- ConversionUtils.decodeBase64Url("script", encoded).toProcessingMessage // TODO: shouldn't be URL-safe + encoded <- ScalazCirceUtils.extract[String](config, "parameters", "script") + raw <- ConversionUtils + .decodeBase64Url("script", encoded) + .toProcessingMessage // TODO: shouldn't be URL-safe compiled <- JavascriptScriptEnrichment.compile(raw).toProcessingMessage enrich = JavascriptScriptEnrichment(compiled) } yield enrich).toValidationNel @@ -56,11 +57,7 @@ object JavascriptScriptEnrichmentConfig extends ParseableEnrichment { } -/** - * Companion object for working with JavaScript scripts. - */ object JavascriptScriptEnrichment { - object Variables { private val prefix = "$snowplow31337" // To avoid collisions val In = s"${prefix}In" @@ -68,14 +65,10 @@ object JavascriptScriptEnrichment { } /** - * Appends an invocation to the script and - * then attempts to compile it. - * - * @param script the JavaScript process() - * function as a String + * Appends an invocation to the script and then attempts to compile it. + * @param script the JavaScript process() function as a String */ private[registry] def compile(script: String): Validation[String, Script] = { - // Script mustn't be null if (Option(script).isEmpty) { return "JavaScript script for evaluation is null".fail @@ -101,20 +94,15 @@ object JavascriptScriptEnrichment { } /** - * Run the process function as stored in the CompiledScript - * against the supplied EnrichedEvent. - * - * @param script the JavaScript process() - * function as a CompiledScript - * @param event The enriched event to - * pass into our process function - * @return a Validation boxing either a - * JSON array of contexts on Success, - * or an error String on Failure + * Run the process function as stored in the CompiledScript against the supplied EnrichedEvent. + * @param script the JavaScript process() function as a CompiledScript + * @param event The enriched event to pass into our process function + * @return a Validation boxing either a JSON array of contexts, or an error String */ - implicit val formats = DefaultFormats - private[registry] def process(script: Script, event: EnrichedEvent): Validation[String, List[JObject]] = { - + private[registry] def process( + script: Script, + event: EnrichedEvent + ): Validation[String, List[Json]] = { val cx = Context.enter() val scope = cx.initStandardObjects @@ -122,55 +110,37 @@ object JavascriptScriptEnrichment { scope.put(Variables.In, scope, Context.javaToJS(event, scope)) val retVal = script.exec(cx, scope) if (Option(retVal).isDefined) { - return s"Evaluated JavaScript script should not return a value; returned: [${retVal}]".fail + return s"Evaluated JavaScript script should not return a value; returned: [$retVal]".fail } } catch { case NonFatal(nf) => - return s"Evaluating JavaScript script threw an exception: [${nf}]".fail + return s"Evaluating JavaScript script threw an exception: [$nf]".fail } finally { Context.exit() } Option(scope.get(Variables.Out)) match { case None => Nil.success - case Some(obj) => { - try { - JsonMethods.parse(obj.asInstanceOf[String]) match { - case JArray(elements) => failFastCast(List[JObject](), elements).success - case _ => s"JavaScript script must return an Array; got [${obj}]".fail - } - } catch { - case NonFatal(nf) => - s"Could not convert object returned from JavaScript script to JValue AST: [${nf}]".fail + case Some(obj) => + parse(obj.asInstanceOf[String]) match { + case Right(js) => + js.asArray match { + case Some(array) => array.toList.success + case None => s"JavaScript script must return an Array; got [$obj]".fail + } + case Left(e) => + ("Could not convert object returned from JavaScript script to Json: " + + s"[${e.getMessage}]").fail } - } - } - } - - /** - * Åttempt to fail fast for our cast to List[]. - * - * Taken from http://stackoverflow.com/a/6690611/255627 - */ - import scala.language.higherKinds - private def failFastCast[A: Manifest, T[A] <: Traversable[A]](as: T[A], any: Any) = { - val res = any.asInstanceOf[T[A]] - if (res.isEmpty) res - else { - manifest[A].newArray(1).update(0, res.head) // force exception on wrong type - res } } } /** * Config for an JavaScript script enrichment - * * @param script The compiled script ready for */ -case class JavascriptScriptEnrichment( - script: Script -) extends Enrichment { +final case class JavascriptScriptEnrichment(script: Script) extends Enrichment { /** * Run the process function as stored in the CompiledScript @@ -182,7 +152,7 @@ case class JavascriptScriptEnrichment( * JSON array of contexts on Success, * or an error String on Failure */ - def process(event: EnrichedEvent): Validation[String, List[JObject]] = + def process(event: EnrichedEvent): Validation[String, List[Json]] = JavascriptScriptEnrichment.process(script, event) } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala index 197dc5902..9eaad1a75 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala @@ -18,68 +18,48 @@ import java.net.URI import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.refererparser.scala.{Parser => RefererParser} import com.snowplowanalytics.refererparser.scala.Referer -import org.json4s.{DefaultFormats, JValue} +import io.circe._ import scalaz._ import utils.{ConversionUtils => CU} -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Companion object. Lets us create a - * RefererParserEnrichment from a JValue - */ +/** Companion object. Lets us create a RefererParserEnrichment from a Json */ object RefererParserEnrichment extends ParseableEnrichment { - - implicit val formats = DefaultFormats - - val supportedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "referer_parser", "jsonschema", 1, 0) + val supportedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "referer_parser", "jsonschema", 1, 0) /** - * Creates a RefererParserEnrichment instance from a JValue. - * + * Creates a RefererParserEnrichment instance from a Json. * @param config The referer_parser enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return a configured RefererParserEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[RefererParserEnrichment] = - isParseable(config, schemaKey).flatMap(conf => { + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[RefererParserEnrichment] = + isParseable(config, schemaKey).flatMap { conf => (for { - param <- ScalazJson4sUtils.extract[List[String]](config, "parameters", "internalDomains") + param <- ScalazCirceUtils.extract[List[String]](config, "parameters", "internalDomains") enrich = RefererParserEnrichment(param) } yield enrich).toValidationNel - }) + } } /** * Config for a referer_parser enrichment - * * @param domains List of internal domains */ -case class RefererParserEnrichment( - domains: List[String] -) extends Enrichment { +final case class RefererParserEnrichment(domains: List[String]) extends Enrichment { - /** - * A Scalaz Lens to update the term within - * a Referer object. - */ - private val termLens: Lens[Referer, MaybeString] = Lens.lensu((r, newTerm) => r.copy(term = newTerm), _.term) + /** A Scalaz Lens to update the term within a Referer object. */ + private val termLens: Lens[Referer, Option[String]] = + Lens.lensu((r, newTerm) => r.copy(term = newTerm), _.term) /** - * Extract details about the referer (sic). - * - * Uses the referer-parser library. - * - * @param uri The referer URI to extract - * referer details from - * @param pageHost The host of the current - * page (used to determine - * if this is an internal - * referer) - * @return a Tuple3 containing referer medium, - * source and term, all Strings + * Extract details about the referer (sic). Uses the referer-parser library. + * @param uri The referer URI to extract referer details from + * @param pageHost The host of the current page (used to determine if this is an internal referer) + * @return a Tuple3 containing referer medium, source and term, all Strings */ def extractRefererDetails(uri: URI, pageHost: String): Option[Referer] = for { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala index 943d92b31..1e9186a55 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala @@ -19,42 +19,34 @@ import scala.util.control.NonFatal import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ +import io.circe.syntax._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.DefaultFormats -import org.json4s.JValue -import org.json4s.JsonDSL._ import ua_parser.Parser import ua_parser.Client -import utils.{ConversionUtils, ScalazJson4sUtils} +import utils.{ConversionUtils, ScalazCirceUtils} -/** - * Companion object. Lets us create a UaParserEnrichment - * from a JValue. - */ +/** Companion object. Lets us create a UaParserEnrichment from a Json. */ object UaParserEnrichmentConfig extends ParseableEnrichment { - implicit val formats = DefaultFormats - - val supportedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "ua_parser_config", "jsonschema", 1, 0) + val supportedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "ua_parser_config", "jsonschema", 1, 0) private val localRulefile = "./ua-parser-rules.yml" - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[UaParserEnrichment] = { - val c = UaParserEnrichment(None) - isParseable(config, schemaKey).flatMap(conf => { + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[UaParserEnrichment] = + isParseable(config, schemaKey).flatMap { conf => (for { rules <- getCustomRules(conf) } yield UaParserEnrichment(rules)).toValidationNel - }) - } + } - private def getCustomRules(conf: JValue): ValidatedMessage[Option[(URI, String)]] = - if (ScalazJson4sUtils.fieldExists(conf, "parameters", "uri")) { + private def getCustomRules(conf: Json): ValidatedMessage[Option[(URI, String)]] = + if (conf.hcursor.downField("parameters").downField("uri").focus.isDefined) { for { - uri <- ScalazJson4sUtils.extract[String](conf, "parameters", "uri") - db <- ScalazJson4sUtils.extract[String](conf, "parameters", "database") + uri <- ScalazCirceUtils.extract[String](conf, "parameters", "uri") + db <- ScalazCirceUtils.extract[String](conf, "parameters", "database") source <- getUri(uri, db) } yield (source, localRulefile).some } else { @@ -71,13 +63,8 @@ object UaParserEnrichmentConfig extends ParseableEnrichment { .toProcessingMessage } -/** - * Config for an ua_parser_config enrichment - * - * Uses uap-java library to parse client attributes - */ -case class UaParserEnrichment(customRulefile: Option[(URI, String)]) extends Enrichment { - +/** Config for an ua_parser_config enrichment. Uses uap-java library to parse client attributes */ +final case class UaParserEnrichment(customRulefile: Option[(URI, String)]) extends Enrichment { override val filesToCache: List[(URI, String)] = customRulefile.map(List(_)).getOrElse(List.empty) @@ -106,9 +93,7 @@ case class UaParserEnrichment(customRulefile: Option[(URI, String)]) extends Enr parser.leftMap(e => s"Failed to initialize ua parser: [${e.getMessage}]") } - /* - * Adds a period in front of a not-null version element - */ + /** Adds a period in front of a not-null version element */ def prependDot(versionElement: String): String = if (versionElement != null) { "." + versionElement @@ -116,9 +101,7 @@ case class UaParserEnrichment(customRulefile: Option[(URI, String)]) extends Enr "" } - /* - * Prepends space before the versionElement - */ + /** Prepends space before the versionElement */ def prependSpace(versionElement: String): String = if (versionElement != null) { " " + versionElement @@ -126,9 +109,7 @@ case class UaParserEnrichment(customRulefile: Option[(URI, String)]) extends Enr "" } - /* - * Checks for null value in versionElement for family parameter - */ + /** Checks for null value in versionElement for family parameter */ def checkNull(versionElement: String): String = if (versionElement == null) { "" @@ -137,54 +118,50 @@ case class UaParserEnrichment(customRulefile: Option[(URI, String)]) extends Enr } /** - * Extracts the client attributes - * from a useragent string, using - * UserAgentEnrichment. - * - * @param useragent The useragent - * String to extract from. - * Should be encoded (i.e. - * not previously decoded). - * @return the json or - * the message of the - * exception, boxed in a - * Scalaz Validation + * Extracts the client attributes from a useragent string, using UserAgentEnrichment. + * @param useragent to extract from. Should be encoded, i.e. not previously decoded. + * @return the json or the message of the exception, boxed in a Scalaz Validation */ - def extractUserAgent(useragent: String): Validation[String, JsonAST.JObject] = + def extractUserAgent(useragent: String): Validation[String, Json] = for { parser <- uaParser c <- try { parser.parse(useragent).success } catch { - case NonFatal(e) => s"Exception parsing useragent [${useragent}]: [${e.getMessage}]".fail + case NonFatal(e) => s"Exception parsing useragent [$useragent]: [${e.getMessage}]".fail } } yield assembleContext(c) - /** - * Assembles ua_parser_context from a parsed user agent. - */ - def assembleContext(c: Client): JsonAST.JObject = { + /** Assembles ua_parser_context from a parsed user agent. */ + def assembleContext(c: Client): Json = { // To display useragent version val useragentVersion = checkNull(c.userAgent.family) + prependSpace(c.userAgent.major) + prependDot( c.userAgent.minor) + prependDot(c.userAgent.patch) // To display operating system version - val osVersion = checkNull(c.os.family) + prependSpace(c.os.major) + prependDot(c.os.minor) + prependDot(c.os.patch) + prependDot( - c.os.patchMinor) - - (("schema" -> "iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0") ~ - ("data" -> - ("useragentFamily" -> c.userAgent.family) ~ - ("useragentMajor" -> c.userAgent.major) ~ - ("useragentMinor" -> c.userAgent.minor) ~ - ("useragentPatch" -> c.userAgent.patch) ~ - ("useragentVersion" -> useragentVersion) ~ - ("osFamily" -> c.os.family) ~ - ("osMajor" -> c.os.major) ~ - ("osMinor" -> c.os.minor) ~ - ("osPatch" -> c.os.patch) ~ - ("osPatchMinor" -> c.os.patchMinor) ~ - ("osVersion" -> osVersion) ~ - ("deviceFamily" -> c.device.family))) + val osVersion = checkNull(c.os.family) + prependSpace(c.os.major) + prependDot(c.os.minor) + + prependDot(c.os.patch) + prependDot(c.os.patchMinor) + + def getJson(s: String): Json = + Option(s).map(Json.fromString).getOrElse(Json.Null) + + Json.obj( + "schema" := + Json.fromString("iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0"), + "data" := Json.obj( + "useragentFamily" := getJson(c.userAgent.family), + "useragentMajor" := getJson(c.userAgent.major), + "useragentMinor" := getJson(c.userAgent.minor), + "useragentPatch" := getJson(c.userAgent.patch), + "useragentVersion" := getJson(useragentVersion), + "osFamily" := getJson(c.os.family), + "osMajor" := getJson(c.os.major), + "osMinor" := getJson(c.os.minor), + "osPatch" := getJson(c.os.patch), + "osPatchMinor" := getJson(c.os.patchMinor), + "osVersion" := getJson(osVersion), + "deviceFamily" := getJson(c.device.family) + ) + ) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala index 8dad21d4b..f3f6dd50f 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala @@ -16,18 +16,21 @@ import scala.util.control.NonFatal import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import eu.bitwalker.useragentutils._ -import org.json4s.JValue +import io.circe._ import org.slf4j.LoggerFactory import scalaz._ import Scalaz._ object UserAgentUtilsEnrichmentConfig extends ParseableEnrichment { - - val supportedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "user_agent_utils_config", "jsonschema", 1, 0) + val supportedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "user_agent_utils_config", "jsonschema", 1, 0) private val log = LoggerFactory.getLogger(getClass()) // Creates a UserAgentUtilsEnrichment instance from a JValue - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[UserAgentUtilsEnrichment.type] = { + def parse( + config: Json, + schemaKey: SchemaKey + ): ValidatedNelMessage[UserAgentUtilsEnrichment.type] = { log.warn( s"user_agent_utils enrichment is deprecated. Please visit here for more information: " + s"https://github.com/snowplow/snowplow/wiki/user-agent-utils-enrichment") @@ -36,14 +39,11 @@ object UserAgentUtilsEnrichmentConfig extends ParseableEnrichment { } /** - * Case class to wrap everything we - * can extract from the useragent - * using UserAgentUtils. - * + * Case class to wrap everything we can extract from the useragent using UserAgentUtils. * Not to be declared inside a class Object * http://stackoverflow.com/questions/17270003/why-are-classes-inside-scala-package-objects-dispreferred */ -case class ClientAttributes( +final case class ClientAttributes( // Browser browserName: String, browserFamily: String, @@ -56,30 +56,17 @@ case class ClientAttributes( osManufacturer: String, // Hardware the OS is running on deviceType: String, - deviceIsMobile: Boolean) - -// Object and a case object with the same name + deviceIsMobile: Boolean +) case object UserAgentUtilsEnrichment extends Enrichment { - private val mobileDeviceTypes = Set(DeviceType.MOBILE, DeviceType.TABLET, DeviceType.WEARABLE) /** - * Extracts the client attributes - * from a useragent string, using - * UserAgentUtils. - * - * TODO: rewrite this when we swap - * out UserAgentUtils for ua-parser - * - * @param useragent The useragent - * String to extract from. - * Should be encoded (i.e. - * not previously decoded). - * @return the ClientAttributes or - * the message of the - * exception, boxed in a - * Scalaz Validation + * Extracts the client attributes from a useragent string, using UserAgentUtils. + * TODO: rewrite this when we swap out UserAgentUtils for ua-parser + * @param useragent to extract from. Should be encoded, i.e. not previously decoded. + * @return the ClientAttributes or the message of the exception, boxed in a Scalaz Validation */ def extractClientAttributes(useragent: String): Validation[String, ClientAttributes] = try { @@ -100,6 +87,7 @@ case object UserAgentUtilsEnrichment extends Enrichment { deviceIsMobile = mobileDeviceTypes.contains(os.getDeviceType) ).success } catch { - case NonFatal(e) => "Exception parsing useragent [%s]: [%s]".format(useragent, e.getMessage).fail + case NonFatal(e) => + "Exception parsing useragent [%s]: [%s]".format(useragent, e.getMessage).fail } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala index a329f1d51..6c7056d3e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala @@ -20,63 +20,61 @@ import scala.util.control.NonFatal import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.weather.providers.openweather.OwmCacheClient import com.snowplowanalytics.weather.providers.openweather.Responses._ +import io.circe._ +import io.circe.generic.auto._ +import io.circe.syntax._ import org.joda.time.{DateTime, DateTimeZone} -import org.json4s.{DefaultFormats, Extraction, JObject, JValue} -import org.json4s.JsonDSL._ import scalaz._ import Scalaz._ -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Companion object. Lets us create an WeatherEnrichment instance from a JValue - */ +/** Companion object. Lets us create an WeatherEnrichment instance from a Json */ object WeatherEnrichmentConfig extends ParseableEnrichment { - - implicit val formats = DefaultFormats - val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow.enrichments", "weather_enrichment_config", "jsonschema", 1, 0) - - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[WeatherEnrichment] = + SchemaCriterion( + "com.snowplowanalytics.snowplow.enrichments", + "weather_enrichment_config", + "jsonschema", + 1, + 0) + + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[WeatherEnrichment] = isParseable(config, schemaKey).flatMap { conf => - { - (for { - apiKey <- ScalazJson4sUtils.extract[String](config, "parameters", "apiKey") - cacheSize <- ScalazJson4sUtils.extract[Int](config, "parameters", "cacheSize") - geoPrecision <- ScalazJson4sUtils.extract[Int](config, "parameters", "geoPrecision") - apiHost <- ScalazJson4sUtils.extract[String](config, "parameters", "apiHost") - timeout <- ScalazJson4sUtils.extract[Int](config, "parameters", "timeout") - enrich = WeatherEnrichment(apiKey, cacheSize, geoPrecision, apiHost, timeout) - } yield enrich).toValidationNel - } + (for { + apiKey <- ScalazCirceUtils.extract[String](config, "parameters", "apiKey") + cacheSize <- ScalazCirceUtils.extract[Int](config, "parameters", "cacheSize") + geoPrecision <- ScalazCirceUtils.extract[Int](config, "parameters", "geoPrecision") + apiHost <- ScalazCirceUtils.extract[String](config, "parameters", "apiHost") + timeout <- ScalazCirceUtils.extract[Int](config, "parameters", "timeout") + enrich = WeatherEnrichment(apiKey, cacheSize, geoPrecision, apiHost, timeout) + } yield enrich).toValidationNel } } /** * Contains weather enrichments based on geo coordinates and time - * * @param apiKey weather provider API KEY * @param cacheSize amount of days with prefetched weather - * @param geoPrecision rounder for geo lat/long floating, which allows to use - * more spatial precise weather stamps + * @param geoPrecision rounder for geo lat/long floating, which allows to use more spatially + * precise weather stamps * @param apiHost address of weather provider's API host * @param timeout timeout in seconds to fetch weather from server */ -case class WeatherEnrichment(apiKey: String, cacheSize: Int, geoPrecision: Int, apiHost: String, timeout: Int) - extends Enrichment { - +case class WeatherEnrichment( + apiKey: String, + cacheSize: Int, + geoPrecision: Int, + apiHost: String, + timeout: Int +) extends Enrichment { private lazy val client = OwmCacheClient(apiKey, cacheSize, geoPrecision, apiHost, timeout) private val schemaUri = "iglu:org.openweathermap/weather/jsonschema/1-0-0" - private implicit val formats = DefaultFormats - /** - * Get weather context as JSON for specific event - * Any non-fatal error will return failure and thus whole event will be - * filtered out in future - * + * Get weather context as JSON for specific event. Any non-fatal error will return failure and + * thus whole event will be filtered out in future * @param latitude enriched event optional latitude (probably null) * @param longitude enriched event optional longitude (probably null) * @param time enriched event optional time (probably null) @@ -87,16 +85,16 @@ case class WeatherEnrichment(apiKey: String, cacheSize: Int, geoPrecision: Int, def getWeatherContext( latitude: Option[JFloat], longitude: Option[JFloat], - time: Option[DateTime]): Validation[String, JObject] = + time: Option[DateTime] + ): Validation[String, Json] = try { getWeather(latitude, longitude, time).map(addSchema) } catch { - case NonFatal(exc) => exc.toString.fail + case NonFatal(exc) => exc.getMessage.fail } /** * Get weather stamp as JSON received from OpenWeatherMap and extracted with Scala Weather - * * @param latitude enriched event optional latitude * @param longitude enriched event optional longitude * @param time enriched event optional time @@ -105,28 +103,30 @@ case class WeatherEnrichment(apiKey: String, cacheSize: Int, geoPrecision: Int, private def getWeather( latitude: Option[JFloat], longitude: Option[JFloat], - time: Option[DateTime]): Validation[String, JObject] = + time: Option[DateTime] + ): Validation[String, Json] = (latitude, longitude, time) match { case (Some(lat), Some(lon), Some(t)) => getCachedOrRequest(lat, lon, (t.getMillis / 1000).toInt).flatMap { weatherStamp => val transformedWeather = transformWeather(weatherStamp) - Extraction.decompose(transformedWeather) match { - case obj: JObject => obj.success - case _ => s"Couldn't transform weather object $transformedWeather into JSON".fail // Shouldn't ever happen - } + transformedWeather.asJson.success } - case _ => s"One of required event fields missing. latitude: $latitude, longitude: $longitude, tstamp: $time".fail + case _ => + ("One of required event fields missing. latitude: " + + s"$latitude, longitude: $longitude, tstamp: $time").fail } /** * Return weather, convert disjunction to validation and stringify error - * * @param latitude event latitude * @param longitude event longitude * @param timestamp event timestamp * @return optional weather stamp */ - private def getCachedOrRequest(latitude: Float, longitude: Float, timestamp: Int): Validation[String, Weather] = + private def getCachedOrRequest( + latitude: Float, + longitude: Float, + timestamp: Int): Validation[String, Weather] = client.getCachedOrRequest(latitude, longitude, timestamp) match { case Right(w) => w.success case Left(e) => e.toString.failure @@ -134,12 +134,14 @@ case class WeatherEnrichment(apiKey: String, cacheSize: Int, geoPrecision: Int, /** * Add Iglu URI to JSON Object - * * @param context weather context as JSON Object * @return JSON Object wrapped as Self-describing JSON */ - private def addSchema(context: JObject): JObject = - ("schema", schemaUri) ~ (("data", context)) + private def addSchema(context: Json): Json = + Json.obj( + "schema" := schemaUri, + "data" := context + ) /** * Apply all necessary transformations (currently only dt(epoch -> db timestamp) @@ -151,7 +153,14 @@ case class WeatherEnrichment(apiKey: String, cacheSize: Int, geoPrecision: Int, */ private[enrichments] def transformWeather(origin: Weather): TransformedWeather = { val time = new DateTime(origin.dt.toLong * 1000, DateTimeZone.UTC).toString - TransformedWeather(origin.main, origin.wind, origin.clouds, origin.rain, origin.snow, origin.weather, time) + TransformedWeather( + origin.main, + origin.wind, + origin.clouds, + origin.rain, + origin.snow, + origin.weather, + time) } } @@ -159,11 +168,12 @@ case class WeatherEnrichment(apiKey: String, cacheSize: Int, geoPrecision: Int, * Copy of `com.snowplowanalytics.weather.providers.openweather.Responses.Weather` intended to * execute typesafe (as opposed to JSON) transformation */ -private[enrichments] case class TransformedWeather( +private[enrichments] final case class TransformedWeather( main: MainInfo, wind: Wind, clouds: Clouds, rain: Option[Rain], snow: Option[Snow], weather: List[WeatherCondition], - dt: String) + dt: String +) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala index 7a92a148d..f7d039cea 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala @@ -42,7 +42,12 @@ object YauaaEnrichment extends ParseableEnrichment { implicit val formats = DefaultFormats val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow.enrichments", "yauaa_enrichment_config", "jsonschema", 1, 0) + SchemaCriterion( + "com.snowplowanalytics.snowplow.enrichments", + "yauaa_enrichment_config", + "jsonschema", + 1, + 0) /** * Creates a YauaaEnrichment instance from a JValue containing the configuration of the enrichment. @@ -54,15 +59,16 @@ object YauaaEnrichment extends ParseableEnrichment { */ def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[YauaaEnrichment] = isParseable(config, schemaKey).flatMap { _ => - val maybeCacheSize = ScalazJson4sUtils.extract[Int](config, "parameters", "cacheSize").toOption + val maybeCacheSize = + ScalazJson4sUtils.extract[Int](config, "parameters", "cacheSize").toOption YauaaEnrichment(maybeCacheSize).success } /** Helper to decapitalize a string. Used for the names of the fields returned in the context. */ def decapitalize(s: String): String = s match { - case _ if s.isEmpty => s + case _ if s.isEmpty => s case _ if s.length == 1 => s.toLowerCase - case _ => s.charAt(0).toLower + s.substring(1) + case _ => s.charAt(0).toLower + s.substring(1) } } @@ -92,7 +98,7 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { val contextSchema = "iglu:nl.basjes/yauaa_context/jsonschema/1-0-0" val defaultDeviceClass = "UNKNOWN" - val defaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> defaultDeviceClass) + val defaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> defaultDeviceClass) /** * Gets the result of YAUAA user agent analysis as self-describing JSON, for a specific event. @@ -105,7 +111,7 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { val parsed = parseUserAgent(userAgent) Extraction.decompose(parsed) match { case obj: JObject => addSchema(obj).success - case _ => s"Couldn't transform YAUAA fields [$parsed] into JSON".failure + case _ => s"Couldn't transform YAUAA fields [$parsed] into JSON".failure } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala index b42b585a3..7a265ba30 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala @@ -16,23 +16,22 @@ package apirequest import java.util.UUID +import cats.syntax.either._ +import com.snowplowanalytics.iglu.client.{JsonSchemaPair, SchemaCriterion, SchemaKey} +import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ +import io.circe.generic.auto._ +import io.circe.jackson._ +import io.circe.syntax._ import scalaz._ import Scalaz._ -import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} -import org.json4s.JsonDSL._ -import org.json4s._ -import org.json4s.jackson.JsonMethods.fromJsonNode import outputs.EnrichedEvent -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Lets us create an ApiRequestEnrichmentConfig from a JValue - */ +/** Lets us create an ApiRequestEnrichmentConfig from a JValue */ object ApiRequestEnrichmentConfig extends ParseableEnrichment { - implicit val formats = DefaultFormats - val supportedSchema = SchemaCriterion( "com.snowplowanalytics.snowplow.enrichments", @@ -44,43 +43,49 @@ object ApiRequestEnrichmentConfig extends ParseableEnrichment { /** * Creates an ApiRequestEnrichment instance from a JValue. - * * @param config The enrichment JSON * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * Must be a supported SchemaKey for this enrichment * @return a configured ApiRequestEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[ApiRequestEnrichment] = + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[ApiRequestEnrichment] = isParseable(config, schemaKey).flatMap(conf => { (for { - inputs <- ScalazJson4sUtils.extract[List[Input]](config, "parameters", "inputs") - httpApi <- ScalazJson4sUtils.extract[HttpApi](config, "parameters", "api", "http") - outputs <- ScalazJson4sUtils.extract[List[Output]](config, "parameters", "outputs") - cache <- ScalazJson4sUtils.extract[Cache](config, "parameters", "cache") + // input ctor throws exception + inputs <- Either.catchNonFatal( + ScalazCirceUtils.extract[List[Input]](config, "parameters", "inputs") + ) match { + case Left(e) => e.getMessage.toProcessingMessage.fail + case Right(r) => r + } + httpApi <- ScalazCirceUtils.extract[HttpApi](config, "parameters", "api", "http") + outputs <- ScalazCirceUtils.extract[List[Output]](config, "parameters", "outputs") + cache <- ScalazCirceUtils.extract[Cache](config, "parameters", "cache") } yield ApiRequestEnrichment(inputs, httpApi, outputs, cache)).toValidationNel }) } -case class ApiRequestEnrichment(inputs: List[Input], api: HttpApi, outputs: List[Output], cache: Cache) - extends Enrichment { - +case class ApiRequestEnrichment( + inputs: List[Input], + api: HttpApi, + outputs: List[Output], + cache: Cache +) extends Enrichment { import ApiRequestEnrichment._ /** - * Primary function of the enrichment - * Failure means HTTP failure, failed unexpected JSON-value, etc - * Successful None skipped lookup (missing key for eg.) - * + * Primary function of the enrichment. Failure means HTTP failure, failed unexpected JSON-value, + * etc. Successful None skipped lookup (missing key for eg.) * @param event currently enriching event * @param derivedContexts derived contexts * @return none if some inputs were missing, validated JSON context if lookup performed */ def lookup( event: EnrichedEvent, - derivedContexts: List[JObject], - customContexts: JsonSchemaPairs, - unstructEvent: JsonSchemaPairs): ValidationNel[String, List[JObject]] = { - + derivedContexts: List[Json], + customContexts: List[JsonSchemaPair], + unstructEvent: List[JsonSchemaPair] + ): ValidationNel[String, List[Json]] = { // Note that [[JsonSchemaPairs]] have specific structure - it is a pair, // where first element is [[SchemaKey]], second element is JSON Object // with keys: `data`, `schema` and `hierarchy` and `schema` contains again [[SchemaKey]] @@ -89,19 +94,24 @@ case class ApiRequestEnrichment(inputs: List[Input], api: HttpApi, outputs: List val jsonUnstructEvent = transformRawPairs(unstructEvent).headOption val templateContext = - Input.buildTemplateContext(inputs, event, derivedContexts, jsonCustomContexts, jsonUnstructEvent) + Input.buildTemplateContext( + inputs, + event, + derivedContexts, + jsonCustomContexts, + jsonUnstructEvent) templateContext.flatMap(getOutputs(_).toValidationNel) } /** * Build URI and try to get value for each of [[outputs]] - * * @param validInputs map to build template context - * @return validated list of lookups, whole lookup will be failed if any of - * outputs were failed + * @return validated list of lookups, whole lookup will be failed if any of outputs were failed */ - private[apirequest] def getOutputs(validInputs: Option[Map[String, String]]): Validation[String, List[JObject]] = { + private[apirequest] def getOutputs( + validInputs: Option[Map[String, String]] + ): Validation[String, List[Json]] = { val result = for { templateContext <- validInputs.toList url <- api.buildUrl(templateContext).toList @@ -113,7 +123,6 @@ case class ApiRequestEnrichment(inputs: List[Input], api: HttpApi, outputs: List /** * Check cache for URL and perform HTTP request if value wasn't found - * * @param url URL to request * @param output currently processing output * @return validated JObject, in case of success ready to be attached to derived contexts @@ -121,12 +130,13 @@ case class ApiRequestEnrichment(inputs: List[Input], api: HttpApi, outputs: List private[apirequest] def cachedOrRequest( url: String, body: Option[String], - output: Output): Validation[Throwable, JObject] = { + output: Output + ): Validation[Throwable, Json] = { val key = cacheKey(url, body) val value = cache.get(key) match { case Some(cachedResponse) => cachedResponse case None => - val json = api.perform(url, body).flatMap(output.parse) + val json = api.perform(url, body).flatMap(output.parseResponse) cache.put(key, json) json } @@ -134,30 +144,28 @@ case class ApiRequestEnrichment(inputs: List[Input], api: HttpApi, outputs: List } } -/** - * Companion object containing common methods for requests and manipulating data - */ +/** Companion object containing common methods for requests and manipulating data */ object ApiRequestEnrichment { /** - * Transform pairs of schema and node obtained from [[utils.shredder.Shredder]] - * into list of regular self-describing instance representing custom context - * or unstruct event - * + * Transform pairs of schema and node obtained from [[utils.shredder.Shredder]] into list of + * regular self-describing instance representing custom context or unstruct event * @param pairs list of pairs consisting of schema and Json nodes - * @return list of regular JObjects + * @return list of regular Json */ - def transformRawPairs(pairs: JsonSchemaPairs): List[JObject] = + def transformRawPairs(pairs: List[JsonSchemaPair]): List[Json] = pairs.map { case (schema, node) => val uri = schema.toSchemaUri - val data = fromJsonNode(node) - ("schema" -> uri) ~ ("data" -> data \ "data") + val data = jacksonToCirce(node) + Json.obj( + "schema" := Json.fromString(uri), + "data" := data.hcursor.downField("data").focus.getOrElse(data) + ) } /** * Creates an UUID based on url and optional body. - * * @param url URL to query * @param body optional request body * @return UUID that identifies of the request. diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala index ae009eb1b..037b5e6cf 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala @@ -13,28 +13,26 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest import com.twitter.util.SynchronizedLruMap -import org.json4s.JValue +import io.circe._ import org.joda.time.DateTime import scalaz._ /** - * Just LRU cache - * + * LRU cache * @param size amount of objects * @param ttl time in seconds to live */ case class Cache(size: Int, ttl: Int) { // URI -> Validated[JSON] - private val cache = new SynchronizedLruMap[String, (Validation[Throwable, JValue], Int)](size) + private val cache = new SynchronizedLruMap[String, (Validation[Throwable, Json], Int)](size) /** * Get a value if it's not outdated - * * @param url HTTP URL * @return validated JSON as it was returned from API server */ - def get(url: String): Option[Validation[Throwable, JValue]] = + def get(url: String): Option[Validation[Throwable, Json]] = cache.get(url) match { case Some((value, created)) if ttl == 0 => Some(value) case Some((value, created)) => { @@ -50,11 +48,10 @@ case class Cache(size: Int, ttl: Int) { /** * Put a value into cache with current timestamp - * * @param key all inputs Map * @param value context object (with Iglu URI, not just plain JSON) */ - def put(key: String, value: Validation[Throwable, JValue]): Unit = { + def put(key: String, value: Validation[Throwable, Json]): Unit = { val now = (new DateTime().getMillis / 1000).toInt cache.put(key, (value, now)) () @@ -62,7 +59,6 @@ case class Cache(size: Int, ttl: Int) { /** * Get actual size of cache - * * @return number of elements in */ private[apirequest] def actualLoad: Int = cache.size diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala index b81753b3e..416861dac 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala @@ -15,8 +15,7 @@ package enrichments.registry.apirequest import java.net.URLEncoder -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization +import io.circe.syntax._ import scalaz._ import Scalaz._ @@ -24,13 +23,17 @@ import utils.HttpClient /** * API client able to make HTTP requests - * * @param method HTTP method * @param uri URI template * @param authentication auth preferences * @param timeout time in milliseconds after which request can be considered failed */ -case class HttpApi(method: String, uri: String, timeout: Int, authentication: Authentication) { +case class HttpApi( + method: String, + uri: String, + timeout: Int, + authentication: Authentication +) { import HttpApi._ private val authUser = for { @@ -46,20 +49,26 @@ case class HttpApi(method: String, uri: String, timeout: Int, authentication: Au /** * Primary API method, taking kv-context derived from event (POJO and contexts), * generating request and sending it - * * @param url URL to query * @param body optional request body * @return self-describing JSON ready to be attached to event contexts */ def perform(url: String, body: Option[String] = None): Validation[Throwable, String] = { - val req = HttpClient.buildRequest(url, authUser = authUser, authPassword = authPassword, body, method, None, None) + val req = HttpClient.buildRequest( + url, + authUser = authUser, + authPassword = authPassword, + body, + method, + None, + None + ) HttpClient.getBody(req) } /** * Build URL from URI templates (http://acme.com/{{key1}}/{{key2}} * Context values taken from event will be URL-encoded - * * @param context key-value context to substitute * @return Some request if everything is built correct, * None if some placeholders weren't matched @@ -72,14 +81,12 @@ case class HttpApi(method: String, uri: String, timeout: Int, authentication: Au /** * Build request data body when the method supports this - * * @param context key-value context - * @return Some body data if method supports body, - * None if method does not support body + * @return Some body data if method supports body, None if method does not support body */ private[apirequest] def buildBody(context: Map[String, String]): Option[String] = method match { - case "POST" | "PUT" => Some(Serialization.write(context)(DefaultFormats)) + case "POST" | "PUT" => Some(context.asJson.noSpaces) case "GET" => None } } @@ -88,7 +95,6 @@ object HttpApi { /** * Check if URI still contain any braces (it's impossible for URL-encoded string) - * * @param uri URI generated out of template * @return true if uri contains no curly braces */ @@ -96,11 +102,9 @@ object HttpApi { !(uri.contains('{') || uri.contains('}')) /** - * Replace all keys (within curly braces) inside template `t` - * with corresponding value. + * Replace all keys (within curly braces) inside template `t` with corresponding value. * This function also double checks pair's key contains only allowed characters * (as specified in ALE config schema), otherwise regex can be injected - * * @param t string with placeholders * @param pair key-value pair * @return template with replaced placehoders for pair's key @@ -114,12 +118,9 @@ object HttpApi { /** * Helper class to configure authentication for HTTP API - * * @param httpBasic single possible auth type is http-basic */ case class Authentication(httpBasic: Option[HttpBasic]) -/** - * Container for HTTP Basic auth credentials - */ +/** Container for HTTP Basic auth credentials */ case class HttpBasic(username: Option[String], password: Option[String]) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala index f0ce0d4e1..a33a7bbbd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala @@ -15,10 +15,10 @@ package enrichments.registry.apirequest import scala.util.control.NonFatal +import cats.syntax.either._ +import io.circe._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.compactJson import outputs.EnrichedEvent import utils.JsonPath._ @@ -27,7 +27,6 @@ import utils.JsonPath._ * Container for key with one (and only one) of possible input sources * Basically, represents a key for future template context and way to get value * out of EnrichedEvent, custom context, derived event or unstruct event. - * * @param key extracted key * @param pojo optional POJO source to take stright from `EnrichedEvent` * @param json optional JSON source to take from context or unstruct event @@ -38,9 +37,11 @@ case class Input(key: String, pojo: Option[PojoInput], json: Option[JsonInput]) // Constructor validation for mapping JSON to `Input` instance (pojo, json) match { case (None, None) => - throw new MappingException("API Request Enrichment Input must represent either JSON OR POJO, none present") + throw new Exception( + "API Request Enrichment Input must represent either JSON OR POJO, none present") case (Some(_), Some(_)) => - throw new MappingException("API Request Enrichment Input must represent either JSON OR POJO, both present") + throw new Exception( + "API Request Enrichment Input must represent either JSON OR POJO, both present") case _ => } @@ -53,7 +54,6 @@ case class Input(key: String, pojo: Option[PojoInput], json: Option[JsonInput]) /** * Get key-value pair input from specific `event` for composing - * * @param event currently enriching event * @return template context with empty or with single element this particular input */ @@ -72,48 +72,51 @@ case class Input(key: String, pojo: Option[PojoInput], json: Option[JsonInput]) /** * Get value out of list of JSON contexts - * * @param derived list of self-describing JObjects representing derived contexts * @param custom list of self-describing JObjects representing custom contexts * @param unstruct optional self-describing JObject representing unstruct event * @return template context with empty or with single element this particular input */ - def getFromJson(derived: List[JObject], custom: List[JObject], unstruct: Option[JObject]): TemplateContext = + def getFromJson( + derived: List[Json], + custom: List[Json], + unstruct: Option[Json] + ): TemplateContext = json match { - case Some(jsonInput) => { + case Some(jsonInput) => val validatedJson = jsonInput.field match { - case "derived_contexts" => getBySchemaCriterion(derived, jsonInput.schemaCriterion).successNel + case "derived_contexts" => + getBySchemaCriterion(derived, jsonInput.schemaCriterion).successNel case "contexts" => getBySchemaCriterion(custom, jsonInput.schemaCriterion).successNel - case "unstruct_event" => getBySchemaCriterion(unstruct.toList, jsonInput.schemaCriterion).successNel + case "unstruct_event" => + getBySchemaCriterion(unstruct.toList, jsonInput.schemaCriterion).successNel case other => s"Error: wrong field [$other] passed to Input.getFromJson. Should be one of: derived_contexts, contexts, unstruct_event".failureNel } (validatedJson |@| validatedJsonPath.toValidationNel) { (validJson, jsonPath) => validJson - .map(jsonPath.json4sQuery) // Query context/UE (always valid) + .map(jsonPath.circeQuery) // Query context/UE (always valid) .map(wrapArray) // Check if array .flatMap(stringifyJson) // Transform to valid string .map(v => Map(key -> Tags.LastVal(v))) // Transform to Key-Value } - } case None => emptyTemplateContext } } /** * Describes how to take key from POJO source - * * @param field `EnrichedEvent` object field */ case class PojoInput(field: String) /** - * * @param field where to get this JSON, one of unstruct_event, contexts or derived_contexts * @param schemaCriterion self-describing JSON you are looking for in the given JSON field. - * You can specify only the SchemaVer MODEL (e.g. 1-), MODEL plus REVISION (e.g. 1-1-) etc - * @param jsonPath JSONPath statement to navigate to the field inside the JSON that you want to use as the input + * You can specify only the SchemaVer MODEL (e.g. 1-), MODEL plus REVISION (e.g. 1-1-) etc + * @param jsonPath JSONPath statement to navigate to the field inside the JSON that you want to use + * as the input */ case class JsonInput(field: String, schemaCriterion: String, jsonPath: String) @@ -142,7 +145,6 @@ object Input { /** * Get template context out of input configurations * If any of inputs missing it will return None - * * @param inputs input-configurations with for keys and instructions how to get values * @param event current enriching event * @param derivedContexts list of contexts derived on enrichment process @@ -153,38 +155,33 @@ object Input { def buildTemplateContext( inputs: List[Input], event: EnrichedEvent, - derivedContexts: List[JObject], - customContexts: List[JObject], - unstructEvent: Option[JObject]): TemplateContext = { - + derivedContexts: List[Json], + customContexts: List[Json], + unstructEvent: Option[Json] + ): TemplateContext = { val eventInputs = buildInputsMap(inputs.map(_.getFromEvent(event))) - val jsonInputs = buildInputsMap(inputs.map(_.getFromJson(derivedContexts, customContexts, unstructEvent))) - + val jsonInputs = buildInputsMap( + inputs.map(_.getFromJson(derivedContexts, customContexts, unstructEvent))) eventInputs |+| jsonInputs } /** * Get data out of all JSON contexts matching `schemaCriterion` * If more than one context match schemaCriterion, first will be picked - * * @param contexts list of self-describing JSON contexts attached to event * @param schemaCriterion part of URI * @return first (optional) self-desc JSON matched `schemaCriterion` */ - def getBySchemaCriterion(contexts: List[JObject], schemaCriterion: String): Option[JValue] = + def getBySchemaCriterion(contexts: List[Json], schemaCriterion: String): Option[Json] = criterionMatch(schemaCriterion).flatMap { criterion => val matched = contexts.filter { context => - context.obj.exists { - case ("schema", JString(schema)) => schema.startsWith(criterion) - case _ => false - } + context.hcursor.get[String]("schema").toOption.map(_.startsWith(criterion)).getOrElse(false) } - matched.map(_ \ "data").headOption + matched.map(_.hcursor.downField("data").focus).flatten.headOption } /** * Transform Schema Criterion to plain string without asterisks - * * @param schemaCriterion schema criterion of format "iglu:vendor/name/schematype/1-*-*" * @return schema criterion of format iglu:vendor/name/schematype/1- */ @@ -199,9 +196,8 @@ object Input { /** * Build and merge template context out of list of all inputs - * - * @param kvPairs list of validated optional (empty/single) kv pairs - * derived from POJO and JSON inputs + * @param kvPairs list of validated optional (empty/single) kv pairs derived from POJO and JSON + * inputs * @return validated optional template context */ def buildInputsMap(kvPairs: List[TemplateContext]): TemplateContext = @@ -213,23 +209,20 @@ object Input { /** * Helper function to stringify JValue to URL-friendly format * JValue should be converted to string for further use in URL template with following rules: - * 1. JString -> as is - * 2. JInt/JDouble/JBool/null -> stringify - * 3. JArray -> concatenate with comma ([1,true,"foo"] -> "1,true,foo"). Nested will be flattened - * 4. JObject -> use as is - * + * 1. string -> as is + * 2. number, booleans, nulls -> stringify + * 3. array -> concatenate with comma ([1,true,"foo"] -> "1,true,foo"). Nested will be flattened + * 4. object -> use as is * @param json arbitrary JSON value - * @return some string best represenging JValue or None if there's no way to stringify it + * @return some string best represenging json or None if there's no way to stringify it */ - private def stringifyJson(json: JValue): Option[String] = json match { - case JString(s) => s.some - case JArray(array) => array.map(stringifyJson).mkString(",").some - case obj: JObject => compactJson(obj).some - case JInt(i) => i.toString.some - case JDouble(d) => d.toString.some - case JDecimal(d) => d.toString.some - case JBool(b) => b.toString.some - case JNull => "null".some // TODO: or None? - case JNothing => none - } + private def stringifyJson(json: Json): Option[String] = + json.fold( + "null".some, + _.toString.some, + _.toString.some, + _.some, + _.map(stringifyJson).mkString(",").some, + o => Json.fromJsonObject(o).noSpaces.some + ) } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala index 3ee489790..b388dad9f 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala @@ -13,121 +13,111 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry.apirequest -import scala.util.control.NonFatal - +import io.circe._ +import io.circe.parser._ +import io.circe.syntax._ import scalaz._ import Scalaz._ -import org.json4s.{JNothing, JObject, JValue} -import org.json4s.JsonDSL._ -import org.json4s.jackson.{compactJson, parseJson} import utils.JsonPath.{query, wrapArray} /** - * Base trait for API output format - * Primary intention of these classes is to perform transformation + * Base trait for API output format. Primary intention of these classes is to perform transformation * of API raw output to self-describing JSON instance */ case class Output(schema: String, json: Option[JsonOutput]) { /** - * Transforming raw API response (text) to JSON - * (in future A => JSON) and extracting value by output's path - * + * Transforming raw API response (text) to JSON (in future A => JSON) and extracting value by + * output's path * @param apiResponse response taken from `ApiMethod` * @return parsed extracted JSON */ - def parse(apiResponse: String): Validation[Throwable, JValue] = json match { - case Some(jsonOutput) => jsonOutput.parse(apiResponse) - case output => new InvalidStateException(s"Error: Unknown output [$output]").failure // Cannot happen now + def parseResponse(apiResponse: String): Validation[Throwable, Json] = json match { + case Some(jsonOutput) => jsonOutput.parseResponse(apiResponse) + case output => + new InvalidStateException(s"Error: Unknown output [$output]").failure // Cannot happen now } /** * Extract value specified by output's path - * * @param value parsed API response * @return extracted validated JSON */ - def extract(value: JValue): Validation[Throwable, JValue] = json match { + def extract(value: Json): Validation[Throwable, Json] = json match { case Some(jsonOutput) => jsonOutput.extract(value) - case output => new InvalidStateException(s"Error: Unknown output [$output]").failure // Cannot happen now + case output => + new InvalidStateException(s"Error: Unknown output [$output]").failure // Cannot happen now } /** * Add `schema` (Iglu URI) to parsed instance - * * @param json JValue parsed from API * @return self-describing JSON instance */ - def describeJson(json: JValue): JObject = - ("schema" -> schema) ~ ("data" -> json) + def describeJson(json: Json): Json = + Json.obj( + "schema" := schema, + "data" := json + ) } /** * Common trait for all API output formats - * * @tparam A type of API response (XML, JSON, etc) */ sealed trait ApiOutput[A] { - val path: String /** * Parse raw response into validated Output format (XML, JSON) - * * @param response API response assumed to be JSON * @return validated JSON */ - def parse(response: String): Validation[Throwable, A] + def parseResponse(response: String): Validation[Throwable, A] /** - * Extract value specified by `path` and - * transform to context-ready JSON data - * + * Extract value specified by `path` and transform to context-ready JSON data * @param response parsed API response * @return extracted by `path` value mapped to JSON */ - def extract(response: A): Validation[Throwable, JValue] + def extract(response: A): Validation[Throwable, Json] /** * Try to parse string as JSON and extract value by JSON PAth - * * @param response API response assumed to be JSON * @return validated extracted value */ - def get(response: String): Validation[Throwable, JValue] = + def get(response: String): Validation[Throwable, Json] = for { - validated <- parse(response) + validated <- parseResponse(response) result <- extract(validated) } yield result } /** * Preference for extracting JSON from API output - * * @param jsonPath JSON Path to required value */ -case class JsonOutput(jsonPath: String) extends ApiOutput[JValue] { - +case class JsonOutput(jsonPath: String) extends ApiOutput[Json] { val path = jsonPath /** * Proxy function for `query` which wrap missing value in error - * * @param json JSON value to look in * @return validated found JSON, with absent value treated like failure */ - def extract(json: JValue): Validation[Throwable, JValue] = + def extract(json: Json): Validation[Throwable, Json] = query(path, json).map(wrapArray) match { - case Success(JNothing) => - ValueNotFoundException(s"Error: no values were found by JSON Path [$jsonPath] in [${compactJson(json)}]").failure + case Success(js) if js.asArray.map(_.isEmpty).getOrElse(false) => + ValueNotFoundException( + s"Error: no values were found by JSON Path [$jsonPath] in [${json.noSpaces}]").failure case other => other.leftMap(JsonPathException.apply) } - def parse(response: String): Validation[Throwable, JValue] = - try { - parseJson(response).success - } catch { - case NonFatal(e) => e.failure + def parseResponse(response: String): Validation[Throwable, Json] = + parse(response) match { + case Right(json) => json.success + case Left(e) => e.failure } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala index 102b1283b..51fa3ae15 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala @@ -17,46 +17,35 @@ import java.net.URI import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ -import org.json4s.JValue +import io.circe._ import scalaz._ import Scalaz._ -/** - * Trait inherited by every enrichment config case class - */ +/** Trait inherited by every enrichment config case class */ trait Enrichment { /** - * Gets the list of files the enrichment requires cached locally. - * The default implementation returns an empty list; if an - * enrichment requires files, it must override this method. - * - * @return A list of pairs, where the first entry in the pair - * indicates the (remote) location of the source file and the - * second indicates the local path where the enrichment expects - * to find the file. + * Gets the list of files the enrichment requires cached locally. The default implementation + * returns an empty list; if an enrichment requires files, it must override this method. + * @return A list of pairs, where the first entry in the pair indicates the (remote) location of + * the source file and the second indicates the local path where the enrichment expects to find + * the file. */ def filesToCache: List[(URI, String)] = List.empty } -/** - * Trait to hold helpers relating to enrichment config - */ +/** Trait to hold helpers relating to enrichment config */ trait ParseableEnrichment { - val supportedSchema: SchemaCriterion /** - * Tests whether a JSON is parseable by a - * specific EnrichmentConfig constructor - * + * Tests whether a JSON is parseable by a specific EnrichmentConfig constructor * @param config The JSON - * @param schemaKey The schemaKey which needs - * to be checked + * @param schemaKey The schemaKey which needs to be checked * @return The JSON or an error message, boxed */ - def isParseable(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[JValue] = - if (supportedSchema matches schemaKey) { + def isParseable(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[Json] = + if (supportedSchema.matches(schemaKey)) { config.success } else { ("Schema key %s is not supported. A '%s' enrichment must have schema '%s'.") diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala index 1179b5468..48ae71583 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala @@ -18,9 +18,9 @@ import outputs.EnrichedEvent object Mutators { /** - * This and the next constant maps from a configuration field name to an EnrichedEvent mutator. The structure is such so that - * it preserves type safety, and it can be easily replaced in the future by generated code that will use the configuration as - * input. + * This and the next constant maps from a configuration field name to an EnrichedEvent mutator. + * The structure is such so that it preserves type safety, and it can be easily replaced in the + * future by generated code that will use the configuration as input. */ val ScalarMutators: Map[String, Mutator] = Map( "user_id" -> Mutator( diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala index fe613b135..b1ac805ee 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala @@ -14,48 +14,50 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry package pii -import org.apache.commons.codec.digest.DigestUtils - import scala.collection.JavaConverters._ import scala.collection.mutable.MutableList +import cats.syntax.either._ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode, TextNode} import com.jayway.jsonpath.{Configuration, JsonPath => JJsonPath} import com.jayway.jsonpath.MapFunction import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} -import org.json4s -import org.json4s.{DefaultFormats, Diff, JValue} -import org.json4s.JsonAST._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.JsonMethods.{compact, parse, render} -import org.json4s.jackson.Serialization.write -import org.json4s.Extraction.decompose +import io.circe._ +import io.circe.jackson._ +import io.circe.syntax._ +import org.apache.commons.codec.digest.DigestUtils import scalaz._ import Scalaz._ -import utils.ScalazJson4sUtils.{extract, fieldExists} import outputs.EnrichedEvent +import serializers._ +import utils.ScalazCirceUtils -/** - * Companion object. Lets us create a PiiPseudonymizerEnrichment - * from a JValue. - */ +/** Companion object. Lets us create a PiiPseudonymizerEnrichment from a Json. */ object PiiPseudonymizerEnrichment extends ParseableEnrichment { - implicit val formats = DefaultFormats + new PiiStrategyPseudonymizeSerializer - override val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow.enrichments", "pii_enrichment_config", "jsonschema", 2, 0, 0) + SchemaCriterion( + "com.snowplowanalytics.snowplow.enrichments", + "pii_enrichment_config", + "jsonschema", + 2, + 0, + 0 + ) - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[PiiPseudonymizerEnrichment] = { + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[PiiPseudonymizerEnrichment] = { for { conf <- matchesSchema(config, schemaKey) - emitIdentificationEvent = extract[Boolean](conf, "emitEvent").toOption + emitIdentificationEvent = ScalazCirceUtils + .extract[Boolean](conf, "emitEvent") + .toOption .getOrElse(false) - piiFields <- extract[List[JObject]](conf, "parameters", "pii").leftMap(_.getMessage) + piiFields <- ScalazCirceUtils + .extract[List[Json]](conf, "parameters", "pii") + .leftMap(_.getMessage) piiStrategy <- extractStrategy(config) piiFieldList <- extractFields(piiFields) } yield PiiPseudonymizerEnrichment(piiFieldList, emitIdentificationEvent, piiStrategy) @@ -72,16 +74,21 @@ object PiiPseudonymizerEnrichment extends ParseableEnrichment { case fName => s"Unknown function $fName".failure } - private def extractFields(piiFields: List[JObject]): Validation[String, List[PiiField]] = - piiFields.map { - case field: JObject => - if (fieldExists(field, "pojo")) - extractString(field, "pojo", "field").flatMap(extractPiiScalarField) - else if (fieldExists(field, "json")) extractPiiJsonField(field \ "json") - else - s"PII Configuration: pii field does not include 'pojo' nor 'json' fields. Got: [${compact(field)}]" - .failure[PiiField] - case json => s"PII Configuration: pii field does not contain an object. Got: [${compact(json)}]".failure[PiiField] + private def extractFields(piiFields: List[Json]): Validation[String, List[PiiField]] = + piiFields.map { json => + extractString(json, "pojo", "field") + .flatMap(extractPiiScalarField) + .orElse { + json.hcursor + .downField("json") + .focus + .toSuccess("No json field") + .flatMap(extractPiiJsonField) + } + .orElse { + ("PII Configuration: pii field does not include 'pojo' nor 'json' fields. " + + s"Got: [${json.noSpaces}]").failure[PiiField] + } }.sequenceU private def extractPiiScalarField(fieldName: String): Validation[String, PiiScalar] = @@ -90,7 +97,7 @@ object PiiPseudonymizerEnrichment extends ParseableEnrichment { .map(PiiScalar(_).success) .getOrElse(s"The specified pojo field $fieldName is not supported".failure) - private def extractPiiJsonField(jsonField: JValue): Validation[String, PiiJson] = { + private def extractPiiJsonField(jsonField: Json): Validation[String, PiiJson] = { val schemaCriterion = extractString(jsonField, "schemaCriterion") .flatMap(sc => SchemaCriterion.parse(sc).leftMap(_.getMessage)) .toValidationNel @@ -108,14 +115,19 @@ object PiiPseudonymizerEnrichment extends ParseableEnrichment { .map(_.success) .getOrElse(s"The specified json field $fieldName is not supported".failure) - private def extractString(jValue: JValue, field: String, tail: String*): Validation[String, String] = - extract[String](jValue, field, tail: _*).leftMap(_.getMessage) + private def extractString( + jValue: Json, + field: String, + tail: String* + ): Validation[String, String] = + ScalazCirceUtils.extract[String](jValue, field, tail: _*).leftMap(_.getMessage) - private def extractStrategy(config: JValue): Validation[String, PiiStrategyPseudonymize] = - extract[PiiStrategyPseudonymize](config, "parameters", "strategy") + private def extractStrategy(config: Json): Validation[String, PiiStrategyPseudonymize] = + ScalazCirceUtils + .extract[PiiStrategyPseudonymize](config, "parameters", "strategy") .leftMap(_.getMessage) - private def matchesSchema(config: JValue, schemaKey: SchemaKey): Validation[String, JValue] = + private def matchesSchema(config: Json, schemaKey: SchemaKey): Validation[String, Json] = if (supportedSchema.matches(schemaKey)) config.success else @@ -128,23 +140,26 @@ object PiiPseudonymizerEnrichment extends ParseableEnrichment { * @param hashFunction the DigestFunction to apply * @param salt salt added to the plain string before hashing */ -final case class PiiStrategyPseudonymize(functionName: String, hashFunction: DigestFunction, salt: String) - extends PiiStrategy { +final case class PiiStrategyPseudonymize( + functionName: String, + hashFunction: DigestFunction, + salt: String +) extends PiiStrategy { val TextEncoding = "UTF-8" override def scramble(clearText: String): String = hash(clearText + salt) def hash(text: String): String = hashFunction(text.getBytes(TextEncoding)) } /** - * The PiiPseudonymizerEnrichment runs after all other enrichments to find fields that are configured as PII (personally - * identifiable information) and apply some anonymization (currently only pseudonymization) on them. Currently a single - * strategy for all the fields is supported due to the configuration format, and there is only one implemented strategy, - * however the enrichment supports a strategy per field. - * - * The user may specify two types of fields in the config `pojo` or `json`. A `pojo` field is effectively a scalar field in the - * EnrichedEvent, whereas a `json` is a "context" formatted field and it can either contain a single value in the case of - * unstruct_event, or an array in the case of derived_events and contexts. - * + * The PiiPseudonymizerEnrichment runs after all other enrichments to find fields that are + * configured as PII (personally identifiable information) and apply some anonymization (currently + * only pseudonymization) on them. Currently a single strategy for all the fields is supported due + * to the configuration format, and there is only one implemented strategy, however the enrichment + * supports a strategy per field. + * The user may specify two types of fields in the config `pojo` or `json`. A `pojo` field is + * effectively a scalar field in the EnrichedEvent, whereas a `json` is a "context" formatted field + * and it can either contain a single value in the case of unstruct_event, or an array in the case + * of derived_events and contexts. * @param fieldList a list of configured PiiFields * @param emitIdentificationEvent whether to emit an identification event * @param strategy the pseudonymization strategy to use @@ -152,22 +167,21 @@ final case class PiiStrategyPseudonymize(functionName: String, hashFunction: Dig case class PiiPseudonymizerEnrichment( fieldList: List[PiiField], emitIdentificationEvent: Boolean, - strategy: PiiStrategy) - extends Enrichment { - implicit val json4sFormats = DefaultFormats + - new PiiModifiedFieldsSerializer + - new PiiStrategyPseudonymizeSerializer - + strategy: PiiStrategy +) extends Enrichment { private val UnstructEventSchema = SchemaKey("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", "1-0-0").toSchemaUri def transformer(event: EnrichedEvent): Unit = { - val modifiedFields: ModifiedFields = fieldList.flatMap(_.transform(event, strategy)) + val modifiedFields = fieldList.flatMap(_.transform(event, strategy)) event.pii = if (emitIdentificationEvent && modifiedFields.nonEmpty) - write( - ("schema" -> UnstructEventSchema) ~ ("data" -> decompose(PiiModifiedFields(modifiedFields, strategy))) - ) + Json + .obj( + "schema" := UnstructEventSchema, + "data" := PiiModifiedFields(modifiedFields, strategy) + ) + .noSpaces else null } } @@ -185,117 +199,137 @@ final case class PiiScalar(fieldMutator: Mutator) extends PiiField { } /** - * Specifies a strategy to use, a field mutator where the JSON can be found in the EnrichedEvent POJO, a schema criterion to - * discriminate which contexts to apply this strategy to, and a JSON path within the contexts where this strategy will - * be applied (the path may correspond to multiple fields). - * + * Specifies a strategy to use, a field mutator where the JSON can be found in the EnrichedEvent + * POJO, a schema criterion to discriminate which contexts to apply this strategy to, and a JSON + * path within the contexts where this strategy will be applied (the path may correspond to + * multiple fields). * @param fieldMutator the field mutator for the JSON field * @param schemaCriterion the schema for which the strategy will be applied * @param jsonPath the path where the strategy will be applied */ -final case class PiiJson(fieldMutator: Mutator, schemaCriterion: SchemaCriterion, jsonPath: String) extends PiiField { - implicit val json4sFormats = DefaultFormats +final case class PiiJson( + fieldMutator: Mutator, + schemaCriterion: SchemaCriterion, + jsonPath: String +) extends PiiField { override def applyStrategy(fieldValue: String, strategy: PiiStrategy): (String, ModifiedFields) = - if (fieldValue != null) { - val (parsedAndSubistuted: JValue, modifiedFields: List[JsonModifiedField]) = parse(fieldValue) match { - case JObject(jObject) => { - val jObjectMap: Map[String, JValue] = jObject.toMap - val contextMapped: Map[String, (JValue, List[JsonModifiedField])] = - jObjectMap.map(mapContextTopFields(_, strategy)) - (JObject(contextMapped.mapValues(_._1).toList), contextMapped.values.map(_._2).flatten) + (for { + value <- Option(fieldValue) + parsed <- parse(value).toOption + (substituted, modifiedFields) = parsed.asObject + .map { obj => + val jObjectMap = obj.toMap + val contextMapped = jObjectMap.map(mapContextTopFields(_, strategy)) + ( + Json.obj(contextMapped.mapValues(_._1).toList: _*), + contextMapped.values.map(_._2).flatten + ) } - case x => (x, List.empty[JsonModifiedField]) - } - val compacted = compact(render(parsedAndSubistuted)) - (compacted, modifiedFields) - } else (null, List.empty[JsonModifiedField]) + .getOrElse((parsed, List.empty[JsonModifiedField])) + } yield (substituted.noSpaces, modifiedFields.toList)).getOrElse((null, List.empty)) - /** - * Map context top fields with strategy if they match. - */ + /** Map context top fields with strategy if they match. */ private def mapContextTopFields( - tuple: (String, json4s.JValue), - strategy: PiiStrategy): (String, (JValue, List[JsonModifiedField])) = tuple match { - case (k: String, contexts: JValue) if k == "data" => - (k, contexts match { - case JArray(contexts) => - val updatedAndModified: List[(JValue, List[JsonModifiedField])] = - contexts.map(getModifiedContext(_, strategy)) - (JArray(updatedAndModified.map(_._1)), updatedAndModified.map(_._2).flatten) - case x => getModifiedContext(x, strategy) + tuple: (String, Json), + strategy: PiiStrategy + ): (String, (Json, List[JsonModifiedField])) = tuple match { + case (k, contexts) if k == "data" => + (k, contexts.asArray match { + case Some(array) => + val updatedAndModified = array.map(getModifiedContext(_, strategy)) + ( + Json.fromValues(updatedAndModified.map(_._1)), + updatedAndModified.map(_._2).flatten.toList + ) + case None => getModifiedContext(contexts, strategy) }) - case (k: String, x: JValue) => (k, (x, List.empty[JsonModifiedField])) + case (k, v) => (k, (v, List.empty[JsonModifiedField])) } - /** - * Returns a modified context or unstruct event along with a list of modified fields. - */ - private def getModifiedContext(jv: JValue, strategy: PiiStrategy): (JValue, List[JsonModifiedField]) = jv match { - case JObject(context) => modifyObjectIfSchemaMatches(context, strategy) - case x => (x, List.empty[JsonModifiedField]) - } + /** Returns a modified context or unstruct event along with a list of modified fields. */ + private def getModifiedContext(jv: Json, strategy: PiiStrategy): (Json, List[JsonModifiedField]) = + jv.asObject + .map { context => + val (obj, fields) = modifyObjectIfSchemaMatches(context.toList, strategy) + (Json.fromJsonObject(obj), fields) + } + .getOrElse((jv, List.empty)) /** - * Tests whether the schema for this event matches the schema criterion and if it does modifies it. + * Tests whether the schema for this event matches the schema criterion and if it does modifies + * it. */ private def modifyObjectIfSchemaMatches( - context: List[(String, json4s.JValue)], - strategy: PiiStrategy): (JObject, List[JsonModifiedField]) = { + context: List[(String, Json)], + strategy: PiiStrategy + ): (JsonObject, List[JsonModifiedField]) = { val fieldsObj = context.toMap (for { schema <- fieldsObj.get("schema") - schemaStr <- schema.extractOpt[String] + schemaStr <- schema.asString parsedSchemaMatches <- SchemaKey.parse(schemaStr).map(schemaCriterion.matches).toOption data <- fieldsObj.get("data") if parsedSchemaMatches updated = jsonPathReplace(data, strategy, schemaStr) - } yield (JObject(fieldsObj.updated("schema", schema).updated("data", updated._1).toList), updated._2)) - .getOrElse((JObject(context), List())) + } yield + ( + JsonObject(fieldsObj.updated("schema", schema).updated("data", updated._1).toList: _*), + updated._2 + )).getOrElse((JsonObject(context: _*), List())) } /** - * Replaces a value in the given context data with the result of applying the strategy that value. + * Replaces a value in the given context with the result of applying the strategy to that value. */ private def jsonPathReplace( - jValue: JValue, + json: Json, strategy: PiiStrategy, - schema: String): (JValue, List[JsonModifiedField]) = { - val objectNode = JsonMethods.mapper.valueToTree[ObjectNode](jValue) + schema: String + ): (Json, List[JsonModifiedField]) = { + val objectNode = io.circe.jackson.mapper.valueToTree[ObjectNode](json) val documentContext = JJsonPath.using(JsonPathConf).parse(objectNode) val modifiedFields = MutableList[JsonModifiedField]() val documentContext2 = documentContext.map( jsonPath, - new ScrambleMapFunction(strategy, modifiedFields, fieldMutator.fieldName, jsonPath, schema)) + new ScrambleMapFunction(strategy, modifiedFields, fieldMutator.fieldName, jsonPath, schema) + ) // make sure it is a structure preserving method, see #3636 - val transformedJValue = JsonMethods.fromJsonNode(documentContext.json[JsonNode]()) - val Diff(_, erroneouslyAdded, _) = jValue diff transformedJValue - val Diff(_, withoutCruft, _) = erroneouslyAdded diff transformedJValue - (withoutCruft, modifiedFields.toList) + //val transformedJValue = JsonMethods.fromJsonNode(documentContext.json[JsonNode]()) + //val Diff(_, erroneouslyAdded, _) = jValue diff transformedJValue + //val Diff(_, withoutCruft, _) = erroneouslyAdded diff transformedJValue + (jacksonToCirce(documentContext2.json[JsonNode]()), modifiedFields.toList) } } -private final class ScrambleMapFunction( +private final case class ScrambleMapFunction( strategy: PiiStrategy, modifiedFields: MutableList[JsonModifiedField], fieldName: String, jsonPath: String, - schema: String) - extends MapFunction { - override def map(currentValue: AnyRef, configuration: Configuration): AnyRef = currentValue match { - case s: String => - val newValue = strategy.scramble(s) - val _ = modifiedFields += JsonModifiedField(fieldName, s, newValue, jsonPath, schema) - newValue - case a: ArrayNode => - a.elements.asScala.map { - case t: TextNode => - val originalValue = t.asText() - val newValue = strategy.scramble(originalValue) - modifiedFields += JsonModifiedField(fieldName, originalValue, newValue, jsonPath, schema) - newValue - case default: AnyRef => default - } - case default: AnyRef => default - } + schema: String +) extends MapFunction { + override def map(currentValue: AnyRef, configuration: Configuration): AnyRef = + currentValue match { + case s: String => + val newValue = strategy.scramble(s) + val _ = modifiedFields += JsonModifiedField(fieldName, s, newValue, jsonPath, schema) + newValue + case a: ArrayNode => + a.elements.asScala.map { + case t: TextNode => + val originalValue = t.asText() + val newValue = strategy.scramble(originalValue) + modifiedFields += JsonModifiedField( + fieldName, + originalValue, + newValue, + jsonPath, + schema + ) + newValue + case default: AnyRef => default + } + case default: AnyRef => default + } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Serializers.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Serializers.scala deleted file mode 100644 index 50bf7257c..000000000 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Serializers.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2017-2019 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.pii - -import org.json4s.JsonDSL._ -import org.json4s.Extraction.decompose -import org.json4s.{CustomSerializer, JObject, MappingException} -import scalaz._ -import Scalaz._ - -/** - * Custom serializer for PiiStrategyPseudonymize class - */ -private[pii] final class PiiStrategyPseudonymizeSerializer - extends CustomSerializer[PiiStrategyPseudonymize](formats => - ({ - case jo: JObject => - implicit val json4sFormats = formats - val function = (jo \ "pseudonymize" \ "hashFunction") - .extractOpt[String] - .toSuccess("Could not get hashFunction from config") - val salt = (jo \ "pseudonymize" \ "salt") - .extractOpt[String] - .toSuccess("Could not get salt from config") - val hashFn = function.flatMap(fn => PiiPseudonymizerEnrichment.getHashFunction(fn)) - (function |@| salt |@| hashFn) { (functionName, salt, functionFn) => - PiiStrategyPseudonymize(functionName, functionFn, salt) - } match { - case Success(psp) => psp - case Failure(msg) => throw new MappingException(msg) - } - }, { - case psp: PiiStrategyPseudonymize => - "pseudonymize" -> ("hashFunction" -> psp.functionName) - })) - -/** - * Custom serializer for PiiModifiedFields class - */ -private[pii] final class PiiModifiedFieldsSerializer - extends CustomSerializer[PiiModifiedFields](formats => { - val PiiTransformationSchema = "iglu:com.snowplowanalytics.snowplow/pii_transformation/jsonschema/1-0-0" - ({ - case jo: JObject => - implicit val json4sFormats = formats - val fields = (jo \ "data" \ "pii").extract[List[ModifiedField]] - val strategy = (jo \ "data" \ "strategy").extract[PiiStrategy] - PiiModifiedFields(fields, strategy) - }, { - case pmf: PiiModifiedFields => - implicit val json4sFormats = formats - ("schema" -> PiiTransformationSchema) ~ - ("data" -> - ("pii" -> decompose( - pmf.modifiedFields.foldLeft(Map.empty[String, List[ModifiedField]]) { - case (m, mf) => - mf match { - case s: ScalarModifiedField => - m + ("pojo" -> (s :: m.getOrElse("pojo", List.empty[ModifiedField]))) - case j: JsonModifiedField => m + ("json" -> (j :: m.getOrElse("json", List.empty[ModifiedField]))) - } - } - )) ~ - ("strategy" -> decompose(pmf.strategy))) - }) - }) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala index 86bf743ab..a209bd156 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala @@ -28,14 +28,15 @@ package object pii { val JsonMutators = Mutators.JsonMutators val ScalarMutators = Mutators.ScalarMutators - // Configuration for JsonPath - // SerializationFeature.FAIL_ON_EMPTY_BEANS is required otherwise an invalid path causes an exception + // Configuration for JsonPath, SerializationFeature.FAIL_ON_EMPTY_BEANS is required otherwise an + // invalid path causes an exception private[pii] val JacksonNodeJsonObjectMapper = { val objectMapper = new ObjectMapper() objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) objectMapper } - // SUPPRESS_EXCEPTIONS is useful here as we prefer an empty list to an exception when a path is not found. + // SUPPRESS_EXCEPTIONS is useful here as we prefer an empty list to an exception when a path is + // not found. private[pii] val JsonPathConf = Configuration .builder() @@ -47,32 +48,34 @@ package object pii { package pii { /** - * PiiStrategy trait. This corresponds to a strategy to apply to a single field. Currently only String input is - * supported. + * PiiStrategy trait. This corresponds to a strategy to apply to a single field. Currently only + * String input is supported. */ trait PiiStrategy { def scramble(clearText: String): String } /** - * The mutator class encapsulates the mutator function and the field name where the mutator will be applied. + * The mutator class encapsulates the mutator function and the field name where the mutator will + * be applied. */ private[pii] final case class Mutator(fieldName: String, muatatorFn: MutatorFn) /** - * Parent class for classes that serialize the values that were modified during the PII enrichment. + * Parent class for classes that serialize the values that were modified during the PII enrichment */ - private[pii] final case class PiiModifiedFields(modifiedFields: ModifiedFields, strategy: PiiStrategy) + private[pii] final case class PiiModifiedFields( + modifiedFields: ModifiedFields, + strategy: PiiStrategy) - /** - * Case class for capturing scalar field modifications. - */ - private[pii] final case class ScalarModifiedField(fieldName: String, originalValue: String, modifiedValue: String) + /** Case class for capturing scalar field modifications. */ + private[pii] final case class ScalarModifiedField( + fieldName: String, + originalValue: String, + modifiedValue: String) extends ModifiedField - /** - * Case class for capturing JSON field modifications. - */ + /** Case class for capturing JSON field modifications. */ private[pii] final case class JsonModifiedField( fieldName: String, originalValue: String, @@ -82,21 +85,21 @@ package pii { extends ModifiedField /** - * PiiField trait. This corresponds to a configuration top-level field (i.e. either a scalar or a JSON field) along with - * a function to apply that strategy to the EnrichedEvent POJO (A scalar field is represented in config py "pojo") + * PiiField trait. This corresponds to a configuration top-level field (i.e. either a scalar or a + * JSON field) along with a function to apply that strategy to the EnrichedEvent POJO (A scalar + * field is represented in config py "pojo") */ private[pii] trait PiiField { /** * The POJO mutator for this field - * * @return fieldMutator */ def fieldMutator: Mutator /** - * Gets an enriched event from the enrichment manager and modifies it according to the specified strategy. - * + * Gets an enriched event from the enrichment manager and modifies it according to the specified + * strategy. * @param event The enriched event */ def transform(event: EnrichedEvent, strategy: PiiStrategy): ModifiedFields = @@ -106,13 +109,12 @@ package pii { } /** - * The modified field trait represents an item that is transformed in either the JSON or a scalar mutators. + * The modified field trait represents an item that is transformed in either the JSON or a scalar + * mutators. */ sealed trait ModifiedField - /** - * Abstract type to get salt using the supported methods - */ + /** Abstract type to get salt using the supported methods */ private[pii] trait PiiStrategyPseudonymizeSalt { def getSalt: String } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala new file mode 100644 index 000000000..13d11eaf9 --- /dev/null +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017-2019 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.pii + +import io.circe._ +import io.circe.generic.auto._ +import io.circe.syntax._ +import scalaz._ +import Scalaz._ + +object serializers { + implicit val piiModifiedFieldsEncoder: Encoder[PiiModifiedFields] = + new Encoder[PiiModifiedFields] { + val PiiTransformationSchema = + "iglu:com.snowplowanalytics.snowplow/pii_transformation/jsonschema/1-0-0" + final def apply(a: PiiModifiedFields): Json = + Json.obj( + "schema" := PiiTransformationSchema, + "data" := Json.obj( + "pii" := + a.modifiedFields + .foldLeft(Map.empty[String, List[ModifiedField]]) { + case (m, mf) => + mf match { + case s: ScalarModifiedField => + m + ("pojo" -> (s :: m.getOrElse("pojo", List.empty[ModifiedField]))) + case j: JsonModifiedField => + m + ("json" -> (j :: m.getOrElse("json", List.empty[ModifiedField]))) + } + } + .asJson + ) + ) + } + + implicit val piiStrategyPseudonymizeDecoder: Decoder[PiiStrategyPseudonymize] = + new Decoder[PiiStrategyPseudonymize] { + final def apply(c: HCursor): Decoder.Result[PiiStrategyPseudonymize] = + for { + function <- c.downField("pseudonymize").get[String]("hashFunction") + hashFn <- PiiPseudonymizerEnrichment + .getHashFunction(function) + .toEither + .leftMap(DecodingFailure(_, List.empty)) + salt <- c.downField("pseudonymize").get[String]("salt") + } yield PiiStrategyPseudonymize(function, hashFn, salt) + } +} diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Cache.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Cache.scala index 50714be6c..397519766 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Cache.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Cache.scala @@ -16,8 +16,8 @@ package enrichments.registry.sqlquery import scala.collection.immutable.IntMap import com.twitter.util.SynchronizedLruMap +import io.circe._ import org.joda.time.DateTime -import org.json4s.JObject import Input.ExtractedValue @@ -25,21 +25,20 @@ import Input.ExtractedValue * Just LRU cache * Stores full IntMap with extracted values as keys and * full list Self-describing contexts as values - * * @param size amount of objects * @param ttl time in seconds to live */ case class Cache(size: Int, ttl: Int) { - private val cache = new SynchronizedLruMap[IntMap[ExtractedValue], (ThrowableXor[List[JObject]], Int)](size) + private val cache = + new SynchronizedLruMap[IntMap[ExtractedValue], (ThrowableXor[List[Json]], Int)](size) /** * Get a value if it's not outdated - * * @param key HTTP URL * @return validated JSON as it was fetched from DB if found */ - def get(key: IntMap[ExtractedValue]): Option[ThrowableXor[List[JObject]]] = + def get(key: IntMap[ExtractedValue]): Option[ThrowableXor[List[Json]]] = cache.get(key) match { case Some((value, _)) if ttl == 0 => Some(value) case Some((value, created)) => @@ -54,11 +53,10 @@ case class Cache(size: Int, ttl: Int) { /** * Put a value into cache with current timestamp - * * @param key all inputs Map * @param value context object (with Iglu URI, not just plain JSON) */ - def put(key: IntMap[ExtractedValue], value: ThrowableXor[List[JObject]]): Unit = { + def put(key: IntMap[ExtractedValue], value: ThrowableXor[List[Json]]): Unit = { val now = (new DateTime().getMillis / 1000).toInt cache.put(key, (value, now)) () @@ -66,7 +64,6 @@ case class Cache(size: Int, ttl: Int) { /** * Get actual size of cache - * * @return number of elements in */ private[sqlquery] def actualLoad: Int = cache.size diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Db.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Db.scala index 76cbeaa3d..0714f9620 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Db.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Db.scala @@ -17,39 +17,39 @@ import java.sql._ import scala.collection.immutable.IntMap -import org.json4s.MappingException - import Input.ExtractedValue /** * Class-container for chosen DB's configuration * Exactly one configuration must be provided - * * @param postgresql optional container for PostgreSQL configuration * @param mysql optional container for MySQL configuration */ case class Db(postgresql: Option[PostgresqlDb] = None, mysql: Option[MysqlDb] = None) { private val realDb: Rdbms = (postgresql, mysql) match { case (Some(_), Some(_)) => - throw new MappingException( - "SQL Query Enrichment Configuration: db must represent either postgresql OR mysql. Both present") + throw new Exception( + "SQL Query Enrichment Configuration: db must represent either " + + "postgresql OR mysql. Both present") case (None, None) => - throw new MappingException( - "SQL Query Enrichment Configuration: db must represent either postgresql OR mysql. None present") - case _ => - List(postgresql, mysql).flatten.head + throw new Exception( + "SQL Query Enrichment Configuration: db must represent either " + + "postgresql OR mysql. None present") + case _ => List(postgresql, mysql).flatten.head } /** - * Create PreparedStatement and fill all its placeholders - * This function expects `placeholderMap` contains exact same amount of placeholders - * as `sql`, otherwise it will result in error downstream - * + * Create PreparedStatement and fill all its placeholders. This function expects `placeholderMap` + * contains exact same amount of placeholders as `sql`, otherwise it will result in error + * downstream * @param sql prepared SQL statement with some unfilled placeholders (?-signs) * @param placeholderMap IntMap with input values * @return filled placeholder or error (unlikely) */ - def createStatement(sql: String, placeholderMap: IntMap[ExtractedValue]): ThrowableXor[PreparedStatement] = + def createStatement( + sql: String, + placeholderMap: IntMap[ExtractedValue] + ): ThrowableXor[PreparedStatement] = realDb.createEmptyStatement(sql).map { preparedStatement => placeholderMap.foreach { case (index, value) => @@ -58,9 +58,7 @@ case class Db(postgresql: Option[PostgresqlDb] = None, mysql: Option[MysqlDb] = preparedStatement } - /** - * Get amount of placeholders (?-signs) in Prepared Statement - */ + /** Get amount of placeholders (?-signs) in Prepared Statement */ def getPlaceholderCount(sql: String): ThrowableXor[Int] = realDb.createEmptyStatement(sql).flatMap(realDb.getPlaceholderCount) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala index bd0fab016..59bc4b9a9 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala @@ -18,10 +18,11 @@ import java.sql.PreparedStatement import scala.collection.immutable.IntMap import scala.util.control.NonFatal +import cats.syntax.either._ +import io.circe._ import io.gatling.jsonpath.JsonPath import scalaz._ import Scalaz._ -import org.json4s._ import utils.JsonPath._ import outputs.EnrichedEvent @@ -30,7 +31,6 @@ import outputs.EnrichedEvent * Container for key with one (and only one) of possible input sources * Basically, represents a key for future template context and way to get value * out of EnrichedEvent, custom context, derived event or unstruct event. - * * @param placeholder extracted key * @param pojo optional pojo source to take straight from EnrichedEvent * @param json optional JSON source to take from context or unstruct event @@ -41,9 +41,9 @@ case class Input(placeholder: Int, pojo: Option[PojoInput], json: Option[JsonInp // Constructor validation for mapping JSON to `Input` instance (pojo, json) match { case (None, None) => - throw new MappingException("Input must represent either json OR pojo, none present") + throw new Exception("Input must represent either json OR pojo, none present") case (Some(_), Some(_)) => - throw new MappingException("Input must represent either json OR pojo, both present") + throw new Exception("Input must represent either json OR pojo, both present") case _ => () } @@ -56,80 +56,81 @@ case class Input(placeholder: Int, pojo: Option[PojoInput], json: Option[JsonInp /** * Get placeholder/input value pair from specific `event` for composing - * * @param event currently enriching event - * @return validated pair of placeholder's postition and extracted value ready - * to be set on PreparedStatement + * @return validated pair of placeholder's postition and extracted value ready to be set on + * PreparedStatement */ - def getFromEvent(event: EnrichedEvent): ValidationNel[Throwable, (Int, Option[ExtractedValue])] = pojo match { - case Some(pojoInput) => - getFieldType(pojoInput.field) match { - case Some(placeholderType) => - try { - val anyRef = event.getClass.getMethod(pojoInput.field).invoke(event) - val option = Option(anyRef.asInstanceOf[placeholderType.PlaceholderType]) - (placeholder, option.map(placeholderType.Value.apply)).successNel - } catch { - case NonFatal(e) => - InvalidInput("SQL Query Enrichment: Extracting from POJO failed: " + e.toString).failureNel - } - case None => InvalidInput("SQL Query Enrichment: Wrong POJO input field was specified").failureNel - } + def getFromEvent(event: EnrichedEvent): ValidationNel[Throwable, (Int, Option[ExtractedValue])] = + pojo match { + case Some(pojoInput) => + getFieldType(pojoInput.field) match { + case Some(placeholderType) => + try { + val anyRef = event.getClass.getMethod(pojoInput.field).invoke(event) + val option = Option(anyRef.asInstanceOf[placeholderType.PlaceholderType]) + (placeholder, option.map(placeholderType.Value.apply)).successNel + } catch { + case NonFatal(e) => + InvalidInput("SQL Query Enrichment: Extracting from POJO failed: " + e.toString).failureNel + } + case None => + InvalidInput("SQL Query Enrichment: Wrong POJO input field was specified").failureNel + } - case None => (placeholder, none).successNel - } + case None => (placeholder, none).successNel + } /** * Get placeholder-value pair input from list of JSON contexts - * * @param derived list of self-describing JObjects representing derived contexts * @param custom list of self-describing JObjects representing custom contexts * @param unstruct optional self-describing JObject representing unstruct event - * @return validated pair of placeholder's postition and extracted value ready - * to be setted on PreparedStatement + * @return validated pair of placeholder's postition and extracted value ready to be setted on + * PreparedStatement */ def getFromJson( - derived: List[JObject], - custom: List[JObject], - unstruct: Option[JObject]): ValidationNel[Throwable, (Int, Option[ExtractedValue])] = + derived: List[Json], + custom: List[Json], + unstruct: Option[Json] + ): ValidationNel[Throwable, (Int, Option[ExtractedValue])] = json match { case Some(jsonInput) => - jsonInput.extract(derived, custom, unstruct).map(json => (placeholder, json.flatMap(extractFromJson))) + jsonInput + .extract(derived, custom, unstruct) + .map(json => (placeholder, json.flatMap(extractFromJson))) case None => (placeholder, none).successNel } } /** * Describes how to take key from POJO source - * * @param field `EnrichedEvent` object field */ case class PojoInput(field: String) /** - * * @param field where to get this json, one of unstruct_event, contexts or derived_contexts * @param schemaCriterion self-describing JSON you are looking for in the given JSON field. - * You can specify only the SchemaVer MODEL (e.g. 1-), MODEL plus REVISION (e.g. 1-1-) etc - * @param jsonPath JSON Path statement to navigate to the field inside the JSON that you want to use as the input + * You can specify only the SchemaVer MODEL (e.g. 1-), MODEL plus REVISION (e.g. 1-1-) etc + * @param jsonPath JSON Path statement to navigate to the field inside the JSON that you want to + * use as the input */ case class JsonInput(field: String, schemaCriterion: String, jsonPath: String) { import Input._ /** * Extract JSON from contexts or unstruct event - * * @param derived list of derived contexts * @param custom list of custom contexts * @param unstruct optional unstruct event - * @return validated optional JSON - * failure means fatal error which should abort enrichment - * none means not-found value + * @return validated optional JSON failure means fatal error which should abort enrichment + * none means not-found value */ def extract( - derived: List[JObject], - custom: List[JObject], - unstruct: Option[JObject]): ValidationNel[Throwable, Option[JValue]] = { + derived: List[Json], + custom: List[Json], + unstruct: Option[Json] + ): ValidationNel[Throwable, Option[Json]] = { val validatedJson = field match { case "derived_contexts" => getBySchemaCriterion(derived, schemaCriterion).successNel case "contexts" => getBySchemaCriterion(custom, schemaCriterion).successNel @@ -147,18 +148,17 @@ case class JsonInput(field: String, schemaCriterion: String, jsonPath: String) { (validatedJsonPath.toValidationNel |@| validatedJson) { (jsonPath, validJson) => validJson - .map(jsonPath.json4sQuery) // Query context/UE (always valid) + .map(jsonPath.circeQuery) // Query context/UE (always valid) .map(wrapArray) // Check if array } } } /** - * Companion object, containing common methods for input data manipulation and - * template context building + * Companion object, containing common methods for input data manipulation and template context + * building */ object Input { - private val criterionRegex = "^(iglu:[a-zA-Z0-9-_.]+/[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+/)([1-9][0-9]*|\\*)-((?:0|[1-9][0-9]*)|\\*)-((?:0|[1-9][0-9]*)|\\*)$".r @@ -172,8 +172,8 @@ object Input { .toMap /** - * Map all textual representations of types of EnrichedEvent properties - * to corresponding StatementPlaceholders + * Map all textual representations of types of EnrichedEvent properties to corresponding + * StatementPlaceholders */ val typeHandlersMap = Map( "java.lang.String" -> StringPlaceholder, @@ -188,11 +188,9 @@ object Input { ) /** - * Value extracted from POJO or JSON - * It is wrapped into StatementPlaceholder#Value, because its real type - * is unknown in compile time and all we need is its method - * `.set(preparedStatement: PreparedStatement, placeholder: Int): Unit` - * to fill PreparedStatement + * Value extracted from POJO or JSON. It is wrapped into StatementPlaceholder#Value, because its + * real type is unknown in compile time and all we need is its method + * `.set(preparedStatement: PreparedStatement, placeholder: Int): Unit` to fill PreparedStatement */ type ExtractedValue = StatementPlaceholder#Value @@ -205,25 +203,20 @@ object Input { /** * Get data out of all JSON contexts matching `schemaCriterion` * If more than one context match schemaCriterion, first will be picked - * * @param contexts list of self-describing JSON contexts attached to event * @param schemaCriterion part of URI * @return first (optional) self-desc JSON matched `schemaCriterion` */ - def getBySchemaCriterion(contexts: List[JObject], schemaCriterion: String): Option[JValue] = + def getBySchemaCriterion(contexts: List[Json], schemaCriterion: String): Option[Json] = criterionMatch(schemaCriterion).flatMap { criterion => val matched = contexts.filter { context => - context.obj.exists { - case ("schema", JString(schema)) => schema.startsWith(criterion) - case _ => false - } + context.hcursor.get[String]("schema").toOption.map(_.startsWith(criterion)).getOrElse(false) } - matched.map(_ \ "data").headOption + matched.map(_.hcursor.downField("data").focus).flatten.headOption } /** * Transform Schema Criterion to plain string without asterisks - * * @param schemaCriterion schema criterion of format "iglu:vendor/name/schematype/1-*-*" * @return schema criterion of format iglu:vendor/name/schematype/1- */ @@ -237,25 +230,23 @@ object Input { } /** - * Build IntMap with all sequental input values - * It returns Failure if **any** of inputs were extracted with fatal error - * (not-found is not a fatal error) - * + * Build IntMap with all sequental input values. It returns Failure if **any** of inputs were + * extracted with fatal error (not-found is not a fatal error) * @param inputs list of all [[Input]] objects * @param event POJO of enriched event * @param derivedContexts list of derived contexts * @param customContexts list of custom contexts * @param unstructEvent optional unstructured event - * @return IntMap if all input values were extracted without error, - * non-empty list of errors otherwise + * @return IntMap if all input values were extracted without error, non-empty list of errors + * otherwise */ def buildPlaceholderMap( inputs: List[Input], event: EnrichedEvent, - derivedContexts: List[JObject], - customContexts: List[JObject], - unstructEvent: Option[JObject]): ValidationNel[Throwable, PlaceholderMap] = { - + derivedContexts: List[Json], + customContexts: List[Json], + unstructEvent: Option[Json] + ): ValidationNel[Throwable, PlaceholderMap] = { val eventInputs = inputs.map(_.getFromEvent(event)) val jsonInputs = inputs.map(_.getFromJson(derivedContexts, customContexts, unstructEvent)) @@ -272,9 +263,8 @@ object Input { } /** - * Check if there any gaps in keys of IntMap (like 1,2,4,5) and keys - * contain "1", so they fill all placeholders - * + * Check if there any gaps in keys of IntMap (like 1,2,4,5) and keys contain "1", so they fill + * all placeholders * @param intMap Map with Ints as keys * @return true if Map contains no gaps and has "1" */ @@ -298,25 +288,26 @@ object Input { /** * Extract runtime-typed (wrapped in [[StatementPlaceholder.Value]]) value from JSON * Objects, Arrays and nulls are mapped to None - * * @param json JSON, probably extracted by JSONPath - * @return Some runtime-typed representation of JSON value - * or None if it is object, array, null or JNothing + * @return Some runtime-typed representation of JSON value or None if it is object, array, null */ - def extractFromJson(json: JValue): Option[ExtractedValue] = json match { - case JString(s) => Some(StringPlaceholder.Value(s)) - case JBool(b) => Some(BooleanPlaceholder.Value(b)) - case JInt(int) if int <= Int.MaxValue && int >= Int.MinValue => Some(IntPlaceholder.Value(int.toInt)) - case JInt(long) => Some(LongPlaceholder.Value(long.toLong)) - case JDouble(d) => Some(DoublePlaceholder.Value(d)) - case _ => None // Objects, Arrays and nulls are invalid ("not-found") values - // In API Request Enrichment null is valid value - } + def extractFromJson(json: Json): Option[ExtractedValue] = json.fold( + none, + b => BooleanPlaceholder.Value(b).some, + n => + n.toInt + .map(IntPlaceholder.Value) + .orElse(n.toLong.map(LongPlaceholder.Value)) + .getOrElse(DoublePlaceholder.Value(n.toDouble)) + .some, + s => StringPlaceholder.Value(s).some, + _ => none, + _ => none + ) /** * Get [[StatementPlaceholder]] for specified field * For e.g. "geo_longitude" => [[FloatPlaceholder]] - * * @param field particular property of EnrichedEvent * @return some */ @@ -338,15 +329,12 @@ object Input { /** * Closure that accepts PreparedStatement and returns setter function which * accepts value (one of allowed types) and its position in PreparedStatement - * * @param preparedStatement statement being mutating * @return setter function closed on prepared statement */ protected def getSetter(preparedStatement: PreparedStatement): (Int, PlaceholderType) => Unit - /** - * Path-dependent class wrapping runtime-typed object - */ + /** Path-dependent class wrapping runtime-typed object */ case class Value(x: PlaceholderType) { def set(preparedStatement: PreparedStatement, placeholder: Int): Unit = getSetter(preparedStatement)(placeholder, x) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala index 140cc24e3..03336f460 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala @@ -17,50 +17,44 @@ import java.sql.{ResultSet, ResultSetMetaData} import scala.collection.mutable.ListBuffer +import io.circe._ +import io.circe.parser._ +import io.circe.syntax._ +import org.joda.time.DateTime import scalaz._ import Scalaz._ -import org.joda.time.DateTime -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods.parseOpt /** * Container class for output preferences. * Describes how to transform data fetched from DB into derived contexts - * * @param json JSON-preferences * @param expectedRows specifies amount of expected rows */ case class Output(json: JsonOutput, expectedRows: String) { import Output._ - /** - * `expectedRows` object converted from String - */ + /** `expectedRows` object converted from String */ val expectedRowsMode = expectedRows match { case "EXACTLY_ONE" => ExactlyOne case "AT_MOST_ONE" => AtMostOne case "AT_LEAST_ONE" => AtLeastOne case "AT_LEAST_ZERO" => AtLeastZero case other => - throw new MappingException(s"SQL Query Enrichment: [$other] is unknown value for expectedRows property") + throw new Exception( + s"SQL Query Enrichment: [$other] is unknown value for expectedRows property") } - /** - * `describe` object converted from String - */ + /** `describe` object converted from String */ val describeMode = json.describeMode /** - * Convert list of rows fetched from DB into list (probably empty or - * single-element) of Self-describing JSON objects (contexts) - * Primary function of class - * + * Convert list of rows fetched from DB into list (probably empty or single-element) of + * Self-describing JSON objects (contexts). Primary function of class * @param resultSet rows fetched from DB * @return list of successful Self-describing JSON Objects or error */ - def convert(resultSet: ResultSet): ThrowableXor[List[JObject]] = { - val buffer = ListBuffer.empty[ThrowableXor[JObject]] + def convert(resultSet: ResultSet): ThrowableXor[List[Json]] = { + val buffer = ListBuffer.empty[ThrowableXor[JsonObject]] while (resultSet.next()) { buffer += parse(resultSet) } val parsedJsons = buffer.result().sequenceU resultSet.close() @@ -74,36 +68,50 @@ case class Output(json: JsonOutput, expectedRows: String) { /** * Validate output according to expectedRows and describe * (attach Schema URI) to context according to json.describes. - * * @param jsons list of JSON Objects derived from SQL rows (row is always JSON Object) * @return validated list of described JSONs */ - def envelope(jsons: List[JObject]): ThrowableXor[List[JObject]] = (describeMode, expectedRowsMode) match { - case (AllRows, AtLeastOne) => AtLeastOne.collect(jsons).map(rows => describe(JArray(rows))).map(List(_)) - case (AllRows, AtLeastZero) => AtLeastZero.collect(jsons).map(rows => describe(JArray(rows))).map(List(_)) - case (AllRows, single) => single.collect(jsons).map(_.headOption.map(describe).toList) - case (EveryRow, any) => any.collect(jsons).map(_.map(describe)) - } + def envelope(jsons: List[JsonObject]): ThrowableXor[List[Json]] = + (describeMode, expectedRowsMode) match { + case (AllRows, AtLeastOne) => + AtLeastOne + .collect(jsons) + .map { jobjs => + describe(Json.arr(jobjs.map(Json.fromJsonObject): _*)) + } + .map(List(_)) + case (AllRows, AtLeastZero) => + AtLeastZero + .collect(jsons) + .map { jobjs => + describe(Json.arr(jobjs.map(Json.fromJsonObject): _*)) + } + .map(List(_)) + case (AllRows, single) => + single.collect(jsons).map(_.headOption.map(js => describe(Json.fromJsonObject(js))).toList) + case (EveryRow, any) => any.collect(jsons).map(_.map(js => describe(Json.fromJsonObject(js)))) + } /** * Transform ResultSet35 (single row) fetched from DB into a JSON Object * Each column maps to an Object's key with name transformed by json.propertyNames * And value transformed using [[JsonOutput#getValue]] - * * @param resultSet single column result * @return successful raw JSON Object or throwable in case of error */ - def parse(resultSet: ResultSet): ThrowableXor[JObject] = + def parse(resultSet: ResultSet): ThrowableXor[JsonObject] = json.transform(resultSet) /** * Attach Iglu URI to JSON making it Self-describing JSON data - * * @param data JSON value to describe (object or array) * @return Self-describing JSON object */ - def describe(data: JValue): JObject = - ("schema" -> json.schema) ~ (("data", data)) + def describe(data: Json): Json = + Json.obj( + "schema" := json.schema, + "data" := data + ) } object Output { @@ -137,33 +145,29 @@ object Output { /** * Validate some amount of rows against predefined expectation - * * @param resultSet JSON objects fetched from DB * @return same list of JSON object as right disjunction if amount * of rows matches expectation or [[InvalidDbResponse]] as * left disjunction if amount is lower or higher than expected */ - def collect(resultSet: List[JObject]): ThrowableXor[List[JObject]] + def collect(resultSet: List[JsonObject]): ThrowableXor[List[JsonObject]] } /** - * Exactly one row is expected. - * 0 or 2+ rows will throw an error, causing the entire event to fail processing + * Exactly one row is expected. 0 or 2+ rows will throw an error, causing the entire event to fail + * processing */ case object ExactlyOne extends ExpectedRowsMode { - def collect(resultSet: List[JObject]): ThrowableXor[List[JObject]] = + def collect(resultSet: List[JsonObject]): ThrowableXor[List[JsonObject]] = resultSet match { case List(one) => List(one).right case other => InvalidDbResponse(s"SQL Query Enrichment: exactly one row was expected").left } } - /** - * Either one or zero rows is expected. - * 2+ rows will throw an error - */ + /** Either one or zero rows is expected. 2+ rows will throw an error */ case object AtMostOne extends ExpectedRowsMode { - def collect(resultSet: List[JObject]): ThrowableXor[List[JObject]] = + def collect(resultSet: List[JsonObject]): ThrowableXor[List[JsonObject]] = resultSet match { case List(one) => List(one).right case List() => Nil.right @@ -171,30 +175,26 @@ object Output { } } - /** - * Always successful - */ + /** Always successful */ case object AtLeastZero extends ExpectedRowsMode { - def collect(resultSet: List[JObject]): ThrowableXor[List[JObject]] = + def collect(resultSet: List[JsonObject]): ThrowableXor[List[JsonObject]] = resultSet.right } - /** - * More that 1 rows are expected - * 0 rows will throw an error - */ + /** More that 1 rows are expected 0 rows will throw an error */ case object AtLeastOne extends ExpectedRowsMode { - def collect(resultSet: List[JObject]): ThrowableXor[List[JObject]] = + def collect(resultSet: List[JsonObject]): ThrowableXor[List[JsonObject]] = resultSet match { - case Nil => InvalidDbResponse(s"SQL Query Enrichment: at least one row was expected. 0 given instead").left + case Nil => + InvalidDbResponse(s"SQL Query Enrichment: at least one row was expected. 0 given instead").left case other => other.right } } } /** - * Handles JSON-specific output (actually, nothing here is JSON-specific, - * unlike API Request Enrichment, so all these properties can go into primary + * Handles JSON-specific output (actually, nothing here is JSON-specific, unlike API Request + * Enrichment, so all these properties can go into primary * Output class as they can be used for *any* output) */ case class JsonOutput(schema: String, describes: String, propertyNames: String) { @@ -204,7 +204,7 @@ case class JsonOutput(schema: String, describes: String, propertyNames: String) val describeMode: DescribeMode = describes match { case "ALL_ROWS" => AllRows case "EVERY_ROW" => EveryRow - case p => throw new MappingException(s"Describe [$p] is not allowed") + case p => throw new Exception(s"Describe [$p] is not allowed") } val propertyNameMode = propertyNames match { @@ -214,18 +214,17 @@ case class JsonOutput(schema: String, describes: String, propertyNames: String) case "SNAKE_CASE" => SnakeCase case "LOWER_CASE" => LowerCase case "UPPER_CASE" => UpperCase - case p => throw new MappingException(s"PropertyName [$p] is not allowed") + case p => throw new Exception(s"PropertyName [$p] is not allowed") } /** * Transform fetched from DB row (as ResultSet) into JSON object * All column names are mapped to object keys using propertyNames - * * @param resultSet column fetched from DB - * @return JSON object as right disjunction in case of success - * or throwable as left disjunction in case of any error + * @return JSON object as right disjunction in case of success or throwable as left disjunction in + * case of any error */ - def transform(resultSet: ResultSet): ThrowableXor[JObject] = { + def transform(resultSet: ResultSet): ThrowableXor[JsonObject] = { val fields = for { rsMeta <- getMetaData(resultSet).liftM[ListT] idx <- ListT[ThrowableXor, Int](getColumnCount(rsMeta).map((x: Int) => (1 to x).toList)) @@ -234,145 +233,124 @@ case class JsonOutput(schema: String, describes: String, propertyNames: String) value <- getColumnValue(colType, idx, resultSet).liftM[ListT] } yield propertyNameMode.transform(colLabel) -> value - fields.toList.map((x: List[JField]) => JObject(x)) + fields.toList.map(x => JsonObject(x: _*)) } } object JsonOutput { - /** - * ADT specifying how to transform key names - */ + /** ADT specifying how to transform key names */ sealed trait PropertyNameMode { def transform(key: String): String } - /** - * Some_Column to Some_Column - */ + /** Some_Column to Some_Column */ case object AsIs extends PropertyNameMode { def transform(key: String): String = key } - /** - * some_column to someColumn - */ + /** some_column to someColumn */ case object CamelCase extends PropertyNameMode { def transform(key: String): String = "_([a-z\\d])".r.replaceAllIn(key, _.group(1).toUpperCase) } - /** - * some_column to SomeColumn - */ + /** some_column to SomeColumn */ case object PascalCase extends PropertyNameMode { def transform(key: String): String = "_([a-z\\d])".r.replaceAllIn(key, _.group(1).toUpperCase).capitalize } - /** - * SomeColumn to some_column - */ + /** SomeColumn to some_column */ case object SnakeCase extends PropertyNameMode { def transform(key: String): String = "[A-Z\\d]".r.replaceAllIn(key, "_" + _.group(0).toLowerCase()) } - /** - * SomeColumn to somecolumn - */ + /** SomeColumn to somecolumn */ case object LowerCase extends PropertyNameMode { def transform(key: String): String = key.toLowerCase } - /** - * SomeColumn to SOMECOLUMN - */ + /** SomeColumn to SOMECOLUMN */ case object UpperCase extends PropertyNameMode { def transform(key: String): String = key.toUpperCase } - /** - * Map of datatypes to JSON-generator functions - */ - val resultsetGetters: Map[String, Object => JValue] = Map( - "java.lang.Integer" -> ((obj: Object) => JInt(obj.asInstanceOf[Int])), - "java.lang.Long" -> ((obj: Object) => JInt(obj.asInstanceOf[Long])), - "java.lang.Boolean" -> ((obj: Object) => JBool(obj.asInstanceOf[Boolean])), - "java.lang.Double" -> ((obj: Object) => JDouble(obj.asInstanceOf[Double])), - "java.lang.Float" -> ((obj: Object) => JDouble(obj.asInstanceOf[Float].toDouble)), - "java.lang.String" -> ((obj: Object) => JString(obj.asInstanceOf[String])), - "java.sql.Date" -> ((obj: Object) => JString(new DateTime(obj.asInstanceOf[java.sql.Date]).toString)) + /** Map of datatypes to JSON-generator functions */ + val resultsetGetters: Map[String, Object => Json] = Map( + "java.lang.Integer" -> ((obj: Object) => Json.fromInt(obj.asInstanceOf[Int])), + "java.lang.Long" -> ((obj: Object) => Json.fromLong(obj.asInstanceOf[Long])), + "java.lang.Boolean" -> ((obj: Object) => Json.fromBoolean(obj.asInstanceOf[Boolean])), + "java.lang.Double" -> ((obj: Object) => Json.fromDoubleOrNull(obj.asInstanceOf[Double])), + "java.lang.Float" -> ((obj: Object) => Json.fromDoubleOrNull(obj.asInstanceOf[Float].toDouble)), + "java.lang.String" -> ((obj: Object) => Json.fromString(obj.asInstanceOf[String])), + "java.sql.Date" -> ((obj: Object) => + Json.fromString(new DateTime(obj.asInstanceOf[java.sql.Date]).toString)) ) - /** - * Lift failing ResultSet#getMetaData into scalaz disjunction - * with Throwable as left-side - */ + /** Lift failing ResultSet#getMetaData into scalaz disjunction with Throwable as left-side */ def getMetaData(rs: ResultSet): ThrowableXor[ResultSetMetaData] = - \/ fromTryCatch rs.getMetaData + \/.fromTryCatch(rs.getMetaData) /** - * Lift failing ResultSetMetaData#getColumnCount into scalaz disjunction - * with Throwable as left-side + * Lift failing ResultSetMetaData#getColumnCount into scalaz disjunction with Throwable as + * left-side */ def getColumnCount(rsMeta: ResultSetMetaData): ThrowableXor[Int] = - \/ fromTryCatch rsMeta.getColumnCount + \/.fromTryCatch(rsMeta.getColumnCount) /** - * Lift failing ResultSetMetaData#getColumnLabel into scalaz disjunction - * with Throwable as left-side + * Lift failing ResultSetMetaData#getColumnLabel into scalaz disjunction with Throwable as + * left-side */ def getColumnLabel(column: Int, rsMeta: ResultSetMetaData): ThrowableXor[String] = - \/ fromTryCatch rsMeta.getColumnLabel(column) + \/.fromTryCatch(rsMeta.getColumnLabel(column)) /** - * Lift failing ResultSetMetaData#getColumnClassName into scalaz disjunction - * with Throwable as left-side + * Lift failing ResultSetMetaData#getColumnClassName into scalaz disjunction with Throwable as + * left-side */ def getColumnType(column: Int, rsMeta: ResultSetMetaData): ThrowableXor[String] = - \/ fromTryCatch rsMeta.getColumnClassName(column) + \/.fromTryCatch(rsMeta.getColumnClassName(column)) /** * Get value from ResultSet using column number - * * @param datatype stringified type representing real type * @param columnIdx column's number in table * @param rs result set fetched from DB * @return JSON in case of success or Throwable in case of SQL error */ - def getColumnValue(datatype: String, columnIdx: Int, rs: ResultSet): ThrowableXor[JValue] = + def getColumnValue(datatype: String, columnIdx: Int, rs: ResultSet): ThrowableXor[Json] = for { value <- (\/ fromTryCatch rs.getObject(columnIdx)).map(Option.apply) - } yield value.map(getValue(_, datatype)).getOrElse(JNull) + } yield value.map(getValue(_, datatype)).getOrElse(Json.Null) /** * Transform value from AnyRef using stringified type hint - * * @param anyRef AnyRef extracted from ResultSet * @param datatype stringified type representing AnyRef's real type * @return AnyRef converted to JSON */ - def getValue(anyRef: AnyRef, datatype: String): JValue = - if (anyRef == null) JNull + def getValue(anyRef: AnyRef, datatype: String): Json = + if (anyRef == null) Json.Null else { - val converter = resultsetGetters getOrElse (datatype, parseObject) + val converter = resultsetGetters.getOrElse(datatype, parseObject) converter(anyRef) } /** - * Default method to parse unknown column type - * First try to parse as JSON Object (PostgreSQL JSON doesn't have a loader for JSON) - * if not successful parse as JSON String + * Default method to parse unknown column type. First try to parse as JSON Object (PostgreSQL + * JSON doesn't have a loader for JSON) if not successful parse as JSON String. + * This method has significant disadvantage, since it can parse string "12 books" as JInt(12), + * but I don't know better way to handle PostgreSQL JSON. */ - // This method has significant disadvantage, since it can parse string "12 books" as JInt(12), - // but I don't know better way to handle PostgreSQL JSON - val parseObject: Object => JValue = (obj) => { + val parseObject: Object => Json = (obj) => { val string = obj.toString - parseOpt(string) match { - case Some(json) => json - case None => JString(string) + parse(string) match { + case Right(js) => js + case _ => Json.fromString(string) } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala index cc2b21845..03198f626 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala @@ -97,7 +97,8 @@ case class PostgresqlDb( val connectionString = s"jdbc:postgresql://$host:$port/$database?user=$username&password=$password" ++ (if (sslMode) "&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory" - else "") + else + "") } /** @@ -116,5 +117,6 @@ case class MysqlDb( val connectionString = s"jdbc:mysql://$host:$port/$database?user=$username&password=$password" ++ (if (sslMode) "&useSsl=true&verifyServerCertificate=false" - else "") + else + "") } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala index 6fc9212ed..c680961e7 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala @@ -16,69 +16,86 @@ package sqlquery import scala.collection.immutable.IntMap -import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} +import cats.syntax.either._ +import com.snowplowanalytics.iglu.client.{JsonSchemaPair, SchemaCriterion, SchemaKey} +import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ +import io.circe.generic.auto._ +import io.circe.jackson._ +import io.circe.syntax._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods.fromJsonNode import outputs.EnrichedEvent -import utils.ScalazJson4sUtils +import utils.ScalazCirceUtils -/** - * Lets us create an SqlQueryEnrichmentConfig from a JValue - */ +/** Lets us create an SqlQueryEnrichmentConfig from a Json */ object SqlQueryEnrichmentConfig extends ParseableEnrichment { - implicit val formats = DefaultFormats - val supportedSchema = - SchemaCriterion("com.snowplowanalytics.snowplow.enrichments", "sql_query_enrichment_config", "jsonschema", 1, 0, 0) + SchemaCriterion( + "com.snowplowanalytics.snowplow.enrichments", + "sql_query_enrichment_config", + "jsonschema", + 1, + 0, + 0) /** - * Creates an SqlQueryEnrichment instance from a JValue. - * + * Creates an SqlQueryEnrichment instance from a Json. * @param config The enrichment JSON - * @param schemaKey The SchemaKey provided for the enrichment - * Must be a supported SchemaKey for this enrichment + * @param schemaKey provided for the enrichment, must be supported by this enrichment * @return a configured SqlQueryEnrichment instance */ - def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[SqlQueryEnrichment] = + def parse(config: Json, schemaKey: SchemaKey): ValidatedNelMessage[SqlQueryEnrichment] = isParseable(config, schemaKey).flatMap(conf => { (for { - inputs <- ScalazJson4sUtils.extract[List[Input]](config, "parameters", "inputs") - db <- ScalazJson4sUtils.extract[Db](config, "parameters", "database") - query <- ScalazJson4sUtils.extract[Query](config, "parameters", "query") - output <- ScalazJson4sUtils.extract[Output](config, "parameters", "output") - cache <- ScalazJson4sUtils.extract[Cache](config, "parameters", "cache") + // input ctor throws exception + inputs <- Either.catchNonFatal( + ScalazCirceUtils.extract[List[Input]](config, "parameters", "inputs") + ) match { + case Left(e) => e.getMessage.toProcessingMessage.fail + case Right(r) => r + } + db <- ScalazCirceUtils.extract[Db](config, "parameters", "database") + query <- ScalazCirceUtils.extract[Query](config, "parameters", "query") + // output ctor throws exception + output <- Either.catchNonFatal( + ScalazCirceUtils.extract[Output](config, "parameters", "output") + ) match { + case Left(e) => e.getMessage.toProcessingMessage.fail + case Right(r) => r + } + cache <- ScalazCirceUtils.extract[Cache](config, "parameters", "cache") } yield SqlQueryEnrichment(inputs, db, query, output, cache)).toValidationNel }) } -case class SqlQueryEnrichment(inputs: List[Input], db: Db, query: Query, output: Output, cache: Cache) - extends Enrichment { - +case class SqlQueryEnrichment( + inputs: List[Input], + db: Db, + query: Query, + output: Output, + cache: Cache +) extends Enrichment { import SqlQueryEnrichment._ /** - * Primary function of the enrichment - * Failure means connection failure, failed unexpected JSON-value, etc - * Successful Nil skipped lookup (unfilled placeholder for eg, empty response) - * + * Primary function of the enrichment. Failure means connection failure, failed unexpected + * JSON-value, etc. Successful Nil skipped lookup (unfilled placeholder for eg, empty response) * @param event currently enriching event * @param derivedContexts derived contexts as list of JSON objects * @param customContexts custom contexts as [[JsonSchemaPairs]] - * @param unstructEvent unstructured (self-describing) event as empty or single element [[JsonSchemaPairs]] + * @param unstructEvent unstructured (self-describing) event as empty or single element + * [[JsonSchemaPairs]] * @return Nil if some inputs were missing, validated JSON contexts if lookup performed */ def lookup( event: EnrichedEvent, - derivedContexts: List[JObject], - customContexts: JsonSchemaPairs, - unstructEvent: JsonSchemaPairs - ): ValidationNel[String, List[JObject]] = { - + derivedContexts: List[Json], + customContexts: List[JsonSchemaPair], + unstructEvent: List[JsonSchemaPair] + ): ValidationNel[String, List[Json]] = { val jsonCustomContexts = transformRawPairs(customContexts) val jsonUnstructEvent = transformRawPairs(unstructEvent).headOption @@ -96,13 +113,11 @@ case class SqlQueryEnrichment(inputs: List[Input], db: Db, query: Query, output: } /** - * Get contexts from cache or perform query if nothing found - * and put result into cache - * + * Get contexts from cache or perform query if nothing found and put result into cache * @param intMap IntMap of extracted values * @return validated list of Self-describing contexts */ - def get(intMap: IntMap[Input.ExtractedValue]): ThrowableXor[List[JObject]] = + def get(intMap: IntMap[Input.ExtractedValue]): ThrowableXor[List[Json]] = cache.get(intMap) match { case Some(response) => response case None => @@ -113,12 +128,11 @@ case class SqlQueryEnrichment(inputs: List[Input], db: Db, query: Query, output: /** * Perform SQL query and convert result to JSON object - * - * @param intMap map with values extracted from inputs and ready to - * be set placeholders in prepared statement + * @param intMap map with values extracted from inputs and ready to be set placeholders in + * prepared statement * @return validated list of Self-describing contexts */ - def query(intMap: IntMap[Input.ExtractedValue]): ThrowableXor[List[JObject]] = + def query(intMap: IntMap[Input.ExtractedValue]): ThrowableXor[List[Json]] = for { sqlQuery <- db.createStatement(query.sql, intMap) resultSet <- db.execute(sqlQuery) @@ -128,12 +142,12 @@ case class SqlQueryEnrichment(inputs: List[Input], db: Db, query: Query, output: /** * Transform [[Input.PlaceholderMap]] to None if not enough input values were extracted * This prevents db from start building a statement while not failing event enrichment - * - * @param placeholderMap some IntMap with extracted values or None if it is known - * already that not all values were extracted + * @param placeholderMap some IntMap with extracted values or None if it is known already that not + * all values were extracted * @return Some unchanged value if all placeholder were filled, None otherwise */ - private def allPlaceholdersFilled(placeholderMap: Input.PlaceholderMap): Validated[Input.PlaceholderMap] = + private def allPlaceholdersFilled( + placeholderMap: Input.PlaceholderMap): Validated[Input.PlaceholderMap] = getPlaceholderCount.map { placeholderCount => placeholderMap match { case Some(intMap) if intMap.keys.size == placeholderCount => Some(intMap) @@ -141,10 +155,7 @@ case class SqlQueryEnrichment(inputs: List[Input], db: Db, query: Query, output: } } - /** - * Stored amount of ?-signs in query.sql - * Initialized once - */ + /** Stored amount of ?-signs in query.sql. Initialized once */ private var lastPlaceholderCount: Validation[Throwable, Int] = InvalidStateException("SQL Query Enrichment: placeholderCount hasn't been initialized").failure @@ -161,31 +172,26 @@ case class SqlQueryEnrichment(inputs: List[Input], db: Db, query: Query, output: } } -/** - * Companion object containing common methods for requests and manipulating data - */ +/** Companion object containing common methods for requests and manipulating data */ object SqlQueryEnrichment { - private implicit val formats = DefaultFormats - /** - * Transform pairs of schema and node obtained from [[utils.shredder.Shredder]] - * into list of regular self-describing JObject representing custom context - * or unstructured event. - * If node isn't Self-describing (doesn't contain data key) - * it will be filtered out. - * + * Transform pairs of schema and node obtained from [[utils.shredder.Shredder]] into list of + * regular self-describing JObject representing custom context or unstructured event. + * If node isn't Self-describing (doesn't contain data key) it will be filtered out. * @param pairs list of pairs consisting of schema and Json nodes * @return list of regular JObjects */ - def transformRawPairs(pairs: JsonSchemaPairs): List[JObject] = - pairs.flatMap { + def transformRawPairs(pairs: List[JsonSchemaPair]): List[Json] = + pairs.map { case (schema, node) => val uri = schema.toSchemaUri - val data = fromJsonNode(node) - data \ "data" match { - case JNothing => Nil - case json => (("schema" -> uri) ~ ("data" -> json): JObject) :: Nil + val data = jacksonToCirce(node) + data.hcursor.downField("data").focus.map { json => + Json.obj( + "schema" := Json.fromString(uri), + "data" := json + ) } - } + }.flatten } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala index 201e88036..89b4bd8e1 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala @@ -21,33 +21,19 @@ import Scalaz._ import utils.{ConversionUtils => CU} -/** - * Holds enrichments related to the - * web page's URL, and the document - * object contained within the page. - */ +/** Holds enrichments related to the web page URL, and the document object contained in the page. */ object PageEnrichments { /** - * Extracts the page URI from - * either the collector's referer - * or the appropriate tracker - * variable. Tracker variable - * takes precedence as per #268 - * - * @param fromReferer The - * page URI reported - * as the referer to - * the collector - * @param fromTracker The - * page URI reported - * by the tracker - * @return either the chosen - * page URI, or an - * error, wrapped in a - * Validation + * Extracts the page URI from either the collector's referer* or the appropriate tracker variable. + * Tracker variable takes precedence as per #268 + * @param fromReferer The page URI reported as the referer to the collector + * @param fromTracker The page URI reported by the tracker + * @return either the chosen page URI, or an error, wrapped in a Validation */ - def extractPageUri(fromReferer: Option[String], fromTracker: Option[String]): Validation[String, Option[URI]] = + def extractPageUri( + fromReferer: Option[String], + fromTracker: Option[String]): Validation[String, Option[URI]] = (fromReferer, fromTracker) match { case (Some(r), None) => CU.stringToUri(r) case (None, Some(t)) => CU.stringToUri(t) @@ -58,14 +44,14 @@ object PageEnrichments { /** * Extract the referrer domain user ID and timestamp from the "_sp={{DUID}}.{{TSTAMP}}" * portion of the querystring - * * @param qsMap The querystring converted to a map * @return Validation boxing a pair of optional strings corresponding to the two fields */ - def parseCrossDomain(qsMap: Map[String, String]): Validation[String, (Option[String], Option[String])] = + def parseCrossDomain( + qsMap: Map[String, String]): Validation[String, (Option[String], Option[String])] = qsMap.get("_sp") match { case Some("") => (None, None).success - case Some(sp) => { + case Some(sp) => val crossDomainElements = sp.split("\\.") val duid = CU.makeTsvSafe(crossDomainElements(0)).some @@ -75,7 +61,6 @@ object PageEnrichments { } tstamp.map(duid -> _) - } case None => (None -> None).success } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CljTomcatLoader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CljTomcatLoader.scala index 22a362867..ee91822c2 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CljTomcatLoader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CljTomcatLoader.scala @@ -21,14 +21,11 @@ import Scalaz._ import utils.ConversionUtils /** - * The dedicated loader for events collected by - * the Clojure Collector running on Tomcat. The - * format started as an approximation of the - * CloudFront format, but has now diverged as + * The dedicated loader for events collected by the Clojure Collector running on Tomcat. The + * format started as an approximation of the CloudFront format, but has now diverged as * we add support for POST payloads. */ object CljTomcatLoader extends Loader[String] { - // The encoding used on these logs private val CollectorEncoding = UTF_8 @@ -36,8 +33,7 @@ object CljTomcatLoader extends Loader[String] { private val CollectorName = "clj-tomcat" // Define the regular expression for extracting the fields - // Adapted and evolved from the Clojure Collector's - // regular expression + // Adapted and evolved from the Clojure Collector's regular expression private val CljTomcatRegex = { val w = "[\\s]+" // Whitespace regex val ow = "(?:" + w // Non-capturing optional whitespace begins @@ -63,17 +59,12 @@ object CljTomcatLoader extends Loader[String] { } /** - * Converts the source string into a - * ValidatedMaybeCollectorPayload. - * + * Converts the source string into a ValidatedMaybeCollectorPayload. * @param line A line of data to convert - * @return either a set of validation - * errors or an Option-boxed - * CanonicalInput object, wrapped - * in a Scalaz ValidatioNel. + * @return either a set of validation errors or an Option-boxed CanonicalInput object, wrapped + * in a Scalaz ValidatioNel. */ def toCollectorPayload(line: String): ValidatedMaybeCollectorPayload = { - def build( qs: String, date: String, @@ -83,7 +74,8 @@ object CljTomcatLoader extends Loader[String] { refr: String, objct: String, ct: Option[String], - bdy: Option[String]): ValidatedMaybeCollectorPayload = { + bdy: Option[String] + ): ValidatedMaybeCollectorPayload = { val querystring = parseQuerystring(CloudfrontLoader.toOption(qs), CollectorEncoding) val timestamp = CloudfrontLoader.toTimestamp(date, time) val contentType = (for { @@ -119,25 +111,27 @@ object CljTomcatLoader extends Loader[String] { line match { // A. For a request, to CljTomcat collector <= v0.6.0 case CljTomcatRegex(date, time, _, _, ip, _, _, objct, _, refr, ua, qs, null, null) => - build(qs, date, time, ip, ua, refr, objct, None, None) // API, content type and request body all unavailable - + // API, content type and request body all unavailable + build(qs, date, time, ip, ua, refr, objct, None, None) // B: For a request without body and potentially a content type, to CljTomcat collector >= v0.7.0 // B.1 No body or content type // TODO: really we ought to be matching on "-", not-"-" and not-"-", "-" as well case CljTomcatRegex(date, time, _, _, ip, _, _, objct, _, refr, ua, qs, "-", "-") => - build(qs, date, time, ip, ua, refr, objct, None, None) // API, content type and request body all unavailable + // API, content type and request body all unavailable + build(qs, date, time, ip, ua, refr, objct, None, None) // B.2 No body but has content type case CljTomcatRegex(date, time, _, _, ip, _, _, objct, _, refr, ua, qs, ct, "-") => - build(qs, date, time, ip, ua, refr, objct, ct.some, None) // API and request body unavailable + // API and request body unavailable + build(qs, date, time, ip, ua, refr, objct, ct.some, None) // C: For a request with content type and/or body, to CljTomcat collector >= v0.7.0 // C.1 Not a POST request case CljTomcatRegex(_, _, _, _, _, op, _, _, _, _, _, _, _, _) if op.toUpperCase != "POST" => - s"Operation must be POST, not ${op.toUpperCase}, if request content type and/or body are provided" - .failNel[Option[CollectorPayload]] + (s"Operation must be POST, not ${op.toUpperCase}, if request content type and/or " + + "body are provided").failNel[Option[CollectorPayload]] // C.2 A POST, let's check we can discern API format // TODO: we should check for nulls/"-"s for ct and body below @@ -146,7 +140,8 @@ object CljTomcatLoader extends Loader[String] { // D. Row not recognised case _ => - "Line does not match raw event format for Clojure Collector".failNel[Option[CollectorPayload]] + "Line does not match raw event format for Clojure Collector" + .failNel[Option[CollectorPayload]] } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CloudfrontLoader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CloudfrontLoader.scala index 5863bdb8e..4e2627ae1 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CloudfrontLoader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CloudfrontLoader.scala @@ -26,23 +26,16 @@ import Scalaz._ import utils.ConversionUtils.singleEncodePcts /** - * The dedicated loader for events - * collected by CloudFront. - * - * We support the following CloudFront access - * log formats: - * + * The dedicated loader for events collected by CloudFront. + * We support the following CloudFront access log formats: * 1. Pre-12 Sep 2012 * 2. 12 Sep 2012 - 21 Oct 2013 * 3. 21 Oct 2013 - 29 Apr 2014 - * 4. Potential future updates, provided they - * are solely additive in nature - * + * 4. Potential future updates, provided they are solely additive in nature * For more details on this format, please see: * http://docs.amazonwebservices.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html#LogFileFormat */ object CloudfrontLoader extends Loader[String] { - // The encoding used on CloudFront logs private val CollectorEncoding = UTF_8 @@ -91,58 +84,47 @@ object CloudfrontLoader extends Loader[String] { private val Cf01Jul2014Regex = toRegex(fields01Jul2014, additionalFields = true) /** - * Converts the source string into a - * ValidatedMaybeCollectorPayload. - * + * Converts the source string into a ValidatedMaybeCollectorPayload. * @param line A line of data to convert - * @return either a set of validation - * errors or an Option-boxed - * CanonicalInput object, wrapped - * in a Scalaz ValidatioNel. + * @return either a set of validation errors or an Option-boxed CanonicalInput object, wrapped + * in a Scalaz ValidatioNel. */ def toCollectorPayload(line: String): ValidatedMaybeCollectorPayload = line match { - // 1. Header row case h if (h.startsWith("#Version:") || h.startsWith("#Fields:")) => None.success - // 2. Not a GET request - case CfOriginalPlusAdditionalRegex(_, _, _, _, _, op, _, _, _, _, _, _) if op.toUpperCase != "GET" => - s"Only GET operations supported for CloudFront Collector, not ${op.toUpperCase}".failNel[Option[CollectorPayload]] - + case CfOriginalPlusAdditionalRegex(_, _, _, _, _, op, _, _, _, _, _, _) + if op.toUpperCase != "GET" => + s"Only GET operations supported for CloudFront Collector, not ${op.toUpperCase}" + .failNel[Option[CollectorPayload]] // 3. Row matches original CloudFront format case CfOriginalRegex(date, time, _, _, ip, _, _, objct, _, rfr, ua, qs) => CloudfrontLogLine(date, time, ip, objct, rfr, ua, qs).toValidatedMaybeCollectorPayload - case Cf12Sep2012Regex(date, time, _, _, ip, _, _, objct, _, rfr, ua, qs) => CloudfrontLogLine(date, time, ip, objct, rfr, ua, qs).toValidatedMaybeCollectorPayload - case Cf21Oct2013Regex(date, time, _, _, ip, _, _, objct, _, rfr, ua, qs) => CloudfrontLogLine(date, time, ip, objct, rfr, ua, qs).toValidatedMaybeCollectorPayload - case Cf29Apr2014Regex(date, time, _, _, ip, _, _, objct, _, rfr, ua, qs) => CloudfrontLogLine(date, time, ip, objct, rfr, ua, qs).toValidatedMaybeCollectorPayload - case Cf01Jul2014Regex(date, time, _, _, ip, _, _, objct, _, rfr, ua, qs, forwardedFor) => CloudfrontLogLine(date, time, ip, objct, rfr, ua, qs, forwardedFor).toValidatedMaybeCollectorPayload - // 4. Row not recognised - case _ => "Line does not match CloudFront header or data row formats".failNel[Option[CollectorPayload]] + case _ => + "Line does not match CloudFront header or data row formats".failNel[Option[CollectorPayload]] } /** - * Converts a CloudFront log-format date and - * a time to a timestamp. - * + * Converts a CloudFront log-format date and a time to a timestamp. * @param date The CloudFront log-format date * @param time The CloudFront log-format time - * @return the timestamp as a Joda DateTime - * or an error String, all wrapped in - * a Scalaz Validation + * @return the timestamp as a Joda DateTime or an error String, all wrapped in a Scalaz Validation */ def toTimestamp(date: String, time: String): Validation[String, DateTime] = try { - DateTime.parse("%sT%s+00:00".format(date, time)).success // Construct a UTC ISO date from CloudFront date and time + DateTime + .parse("%sT%s+00:00".format(date, time)) + .success // Construct a UTC ISO date from CloudFront date and time } catch { case NonFatal(e) => "Unexpected exception converting date [%s] and time [%s] to timestamp: [%s]" @@ -151,10 +133,7 @@ object CloudfrontLoader extends Loader[String] { } /** - * Checks whether a String field is a hyphen - * "-", which is used by CloudFront to signal - * a null. - * + * Checks whether a String field is a hyphen "-", which is used by CloudFront to signal a null. * @param field The field to check * @return True if the String was a hyphen "-" */ @@ -165,14 +144,9 @@ object CloudfrontLoader extends Loader[String] { } /** - * 'Cleans' a string to make it parsable by - * URLDecoder.decode. - * - * The '%' character seems to be appended to the - * end of some URLs in the CloudFront logs, causing - * Exceptions when using URLDecoder.decode. Perhaps - * a CloudFront bug? - * + * 'Cleans' a string to make it parsable by URLDecoder.decode. + * The '%' character seems to be appended to the end of some URLs in the CloudFront logs, causing + * Exceptions when using URLDecoder.decode. Perhaps a CloudFront bug? * @param uri The String to clean * @return the cleaned string */ @@ -195,8 +169,8 @@ object CloudfrontLoader extends Loader[String] { rfr: String, ua: String, qs: String, - forwardedFor: String = "-") { - + forwardedFor: String = "-" + ) { def toValidatedMaybeCollectorPayload: ValidatedMaybeCollectorPayload = { // Validations, and let's strip double-encodings val timestamp = toTimestamp(date, time) @@ -213,22 +187,23 @@ object CloudfrontLoader extends Loader[String] { val api = CollectorApi.parse(objct) - (timestamp.toValidationNel |@| querystring.toValidationNel |@| api.toValidationNel) { (t, q, a) => - CollectorPayload( - q, - CollectorName, - CollectorEncoding.toString, - None, // No hostname for CloudFront - Some(t), - toOption(ip), - toOption(userAgent), - referer, - Nil, // No headers for CloudFront - None, // No collector-set user ID for CloudFront - a, // API vendor/version - None, // No content type - None // No request body - ).some + (timestamp.toValidationNel |@| querystring.toValidationNel |@| api.toValidationNel) { + (t, q, a) => + CollectorPayload( + q, + CollectorName, + CollectorEncoding.toString, + None, // No hostname for CloudFront + Some(t), + toOption(ip), + toOption(userAgent), + referer, + Nil, // No headers for CloudFront + None, // No collector-set user ID for CloudFront + a, // API vendor/version + None, // No content type + None // No request body + ).some } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala index 3e6306689..5f78805f7 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala @@ -29,16 +29,18 @@ object IpAddressExtractor { /** * If a request has been forwarded, extract the original client IP address; * otherwise return the standard IP address - * * If both FORWARDED and X-FORWARDED-FOR are set, * the IP contained in X-FORWARDED-FOR will be used. - * * @param headers List of headers potentially containing X-FORWARDED-FOR or FORWARDED * @param lastIp Fallback IP address if no X-FORWARDED-FOR or FORWARDED header exists * @return True client IP address */ @tailrec - def extractIpAddress(headers: List[String], lastIp: String, maybeForwardedForIp: Option[String] = None): String = + def extractIpAddress( + headers: List[String], + lastIp: String, + maybeForwardedForIp: Option[String] = None + ): String = headers match { case h :: t => h.toLowerCase match { @@ -51,14 +53,13 @@ object IpAddressExtractor { case Nil => maybeForwardedForIp match { case Some(forwardedForIp) => forwardedForIp - case _ => lastIp + case _ => lastIp } } /** * If a request has been forwarded, extract the original client IP address; * otherwise return the standard IP address - * * @param xForwardedFor x-forwarded-for field from the Cloudfront log * @param lastIp Fallback IP address if no X-FORWARDED-FOR header exists * @return True client IP address diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala index 1f97aaba8..8c7b40113 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala @@ -23,78 +23,52 @@ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -/** - * Companion object to the CollectorLoader. - * Contains factory methods. - */ +/** Companion object to the CollectorLoader. Contains factory methods. */ object Loader { - private val TsvRegex = "^tsv/(.*)$".r private val NdjsonRegex = "^ndjson/(.*)$".r /** - * Factory to return a CollectorLoader - * based on the supplied collector - * identifier (e.g. "cloudfront" or - * "clj-tomcat"). - * - * @param collector Identifier for the - * event collector - * @return a CollectorLoader object, or - * an an error message, boxed - * in a Scalaz Validation + * Factory to return a CollectorLoader based on the supplied collector identifier (e.g. + * "cloudfront" or "clj-tomcat"). + * @param collector Identifier for the event collector + * @return a CollectorLoader object, or an an error message, boxed in a Scalaz Validation */ - def getLoader(collectorOrProtocol: String): Validation[String, Loader[_]] = collectorOrProtocol match { - case "cloudfront" => CloudfrontLoader.success - case "clj-tomcat" => CljTomcatLoader.success - case "thrift" => ThriftLoader.success // Finally - a data protocol rather than a piece of software - case TsvRegex(f) => TsvLoader(f).success - case NdjsonRegex(f) => NdjsonLoader(f).success - case c => "[%s] is not a recognised Snowplow event collector".format(c).fail - } + def getLoader(collectorOrProtocol: String): Validation[String, Loader[_]] = + collectorOrProtocol match { + case "cloudfront" => CloudfrontLoader.success + case "clj-tomcat" => CljTomcatLoader.success + case "thrift" => + ThriftLoader.success // Finally - a data protocol rather than a piece of software + case TsvRegex(f) => TsvLoader(f).success + case NdjsonRegex(f) => NdjsonLoader(f).success + case c => "[%s] is not a recognised Snowplow event collector".format(c).fail + } } -/** - * All loaders must implement this - * abstract base class. - */ +/** All loaders must implement this abstract base class. */ abstract class Loader[T] { /** - * Converts the source string into a - * CanonicalInput. - * - * TODO: need to change this to - * handling some sort of validation - * object. - * + * Converts the source string into a CanonicalInput. + * TODO: need to change this to handling some sort of validation object. * @param line A line of data to convert - * @return a CanonicalInput object, Option- - * boxed, or None if no input was - * extractable. + * @return a CanonicalInput object, Option-boxed, or None if no input was extractable. */ def toCollectorPayload(line: T): ValidatedMaybeCollectorPayload /** - * Converts a querystring String - * into a non-empty list of NameValuePairs. - * - * Returns a non-empty list of - * NameValuePairs on Success, or a Failure - * String. - * - * @param qs Option-boxed querystring - * String to extract name-value - * pairs from, or None - * @param encoding The encoding used - * by this querystring - * @return either a NonEmptyList of - * NameValuePairs or an error - * message, boxed in a Scalaz - * Validation + * Converts a querystring String into a non-empty list of NameValuePairs. + * Returns a non-empty list of NameValuePairs on Success, or a Failure String. + * @param qs Option-boxed querystring String to extract name-value pairs from, or None + * @param encoding The encoding used by this querystring + * @return either a NEL of NameValuePairs or an error message, boxed in a Scalaz Validation */ - protected[loaders] def parseQuerystring(qs: Option[String], enc: Charset): ValidatedNameValuePairs = qs match { - case Some(q) => { + protected[loaders] def parseQuerystring( + qs: Option[String], + enc: Charset + ): ValidatedNameValuePairs = qs match { + case Some(q) => try { URLEncodedUtils.parse(URI.create("http://localhost/?" + q), enc).toList.success } catch { @@ -103,7 +77,6 @@ abstract class Loader[T] { .format(q, enc, e.getMessage) .fail } - } case None => Nil.success } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/NdjsonLoader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/NdjsonLoader.scala index 185e23c2c..a37aa70e9 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/NdjsonLoader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/NdjsonLoader.scala @@ -13,59 +13,49 @@ package com.snowplowanalytics.snowplow.enrich.common package loaders -import com.fasterxml.jackson.core.JsonParseException +import io.circe.parser._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ -case class NdjsonLoader(adapter: String) extends Loader[String] { +final case class NdjsonLoader(adapter: String) extends Loader[String] { private val CollectorName = "ndjson" private val CollectorEncoding = "UTF-8" /** - * Converts the source string into a - * CanonicalInput. - * + * Converts the source string into a CanonicalInput. * @param line A line of data to convert - * @return a CanonicalInput object, Option- - * boxed, or None if no input was - * extractable. + * @return a CanonicalInput object, Option-boxed, or None if no input was extractable. */ override def toCollectorPayload(line: String): ValidatedMaybeCollectorPayload = - try { - - if (line.replaceAll("\r?\n", "").isEmpty) { - Success(None) - } else if (line.split("\r?\n").size > 1) { - "Too many lines! Expected single line".failNel - } else { - parse(line) - CollectorApi - .parse(adapter) - .map( - CollectorPayload( - Nil, - CollectorName, - CollectorEncoding, - None, - None, - None, - None, - None, - Nil, - None, - _, - None, - Some(line) - ).some - ) - .toValidationNel + if (line.replaceAll("\r?\n", "").isEmpty) { + Success(None) + } else if (line.split("\r?\n").size > 1) { + "Too many lines! Expected single line".failNel + } else { + parse(line) match { + case Left(e) => s"Unparsable JSON: ${e.getMessage}".failNel + case _ => + CollectorApi + .parse(adapter) + .map( + CollectorPayload( + Nil, + CollectorName, + CollectorEncoding, + None, + None, + None, + None, + None, + Nil, + None, + _, + None, + Some(line) + ).some + ) + .toValidationNel } - - } catch { - case e: JsonParseException => "Unparsable JSON".failNel } - } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala index fc13a60fe..70cf12dcd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala @@ -16,9 +16,12 @@ package loaders import java.nio.charset.Charset import scala.collection.JavaConversions._ +import scala.util.control.NonFatal import com.snowplowanalytics.iglu.client.{SchemaCriterion, SchemaKey} -import com.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.{CollectorPayload => CollectorPayload1} +import com.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.{ + CollectorPayload => CollectorPayload1 +} import com.snowplowanalytics.snowplow.SchemaSniffer.thrift.model1.SchemaSniffer import com.snowplowanalytics.snowplow.collectors.thrift.SnowplowRawEvent import org.apache.http.NameValuePair @@ -27,31 +30,25 @@ import org.joda.time.{DateTime, DateTimeZone} import scalaz._ import Scalaz._ -/** - * Loader for Thrift SnowplowRawEvent objects. - */ +/** Loader for Thrift SnowplowRawEvent objects. */ object ThriftLoader extends Loader[Array[Byte]] { - private val thriftDeserializer = new TDeserializer - private val ExpectedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "CollectorPayload", "thrift", 1, 0) + private val ExpectedSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "CollectorPayload", "thrift", 1, 0) /** * Converts the source string into a ValidatedMaybeCollectorPayload. * Checks the version of the raw event and calls the appropriate method. - * - * @param line A serialized Thrift object Byte array mapped to a String. - * The method calling this should encode the serialized object - * with `snowplowRawEventBytes.map(_.toChar)`. - * Reference: http://stackoverflow.com/questions/5250324/ - * @return either a set of validation errors or an Option-boxed - * CanonicalInput object, wrapped in a Scalaz ValidatioNel. + * @param line A serialized Thrift object Byte array mapped to a String. The method calling this + * should encode the serialized object with `snowplowRawEventBytes.map(_.toChar)`. + * Reference: http://stackoverflow.com/questions/5250324/ + * @return either a set of validation errors or an Option-boxed CanonicalInput object, wrapped in + * a Scalaz ValidatioNel. */ def toCollectorPayload(line: Array[Byte]): ValidatedMaybeCollectorPayload = try { - var schema = new SchemaSniffer - this.synchronized { thriftDeserializer.deserialize( schema, @@ -61,7 +58,6 @@ object ThriftLoader extends Loader[Array[Byte]] { if (schema.isSetSchema) { val actualSchema = SchemaKey.parse(schema.getSchema).leftMap(_.toString).toValidationNel - for { as <- actualSchema res <- if (ExpectedSchema.matches(as)) { @@ -70,29 +66,25 @@ object ThriftLoader extends Loader[Array[Byte]] { s"Verifying record as $ExpectedSchema failed: found $as".failNel } } yield res - } else { convertOldSchema(line) } } catch { // TODO: Check for deserialization errors. - case e: Throwable => + case NonFatal(e) => s"Error deserializing raw event: ${e.getMessage}".failNel[Option[CollectorPayload]] } /** * Converts the source string into a ValidatedMaybeCollectorPayload. * Assumes that the byte array is a serialized CollectorPayload, version 1. - * - * @param line A serialized Thrift object Byte array mapped to a String. - * The method calling this should encode the serialized object - * with `snowplowRawEventBytes.map(_.toChar)`. - * Reference: http://stackoverflow.com/questions/5250324/ - * @return either a set of validation errors or an Option-boxed - * CanonicalInput object, wrapped in a Scalaz ValidatioNel. + * @param line A serialized Thrift object Byte array mapped to a String. The method calling this + * should encode the serialized object with`snowplowRawEventBytes.map(_.toChar)`. + * Reference: http://stackoverflow.com/questions/5250324/ + * @return either a set of validation errors or an Option-boxed CanonicalInput object, wrapped in + * a Scalaz ValidatioNel. */ private def convertSchema1(line: Array[Byte]): ValidatedMaybeCollectorPayload = { - var collectorPayload = new CollectorPayload1 this.synchronized { thriftDeserializer.deserialize( @@ -141,19 +133,15 @@ object ThriftLoader extends Loader[Array[Byte]] { } /** - * Converts the source string into a ValidatedMaybeCollectorPayload. - * Assumes that the byte array is an old serialized SnowplowRawEvent - * which is not self-describing. - * - * @param line A serialized Thrift object Byte array mapped to a String. - * The method calling this should encode the serialized object - * with `snowplowRawEventBytes.map(_.toChar)`. - * Reference: http://stackoverflow.com/questions/5250324/ - * @return either a set of validation errors or an Option-boxed - * CanonicalInput object, wrapped in a Scalaz ValidatioNel. + * Converts the source string into a ValidatedMaybeCollectorPayload. Assumes that the byte array + * is an old serialized SnowplowRawEvent which is not self-describing. + * @param line A serialized Thrift object Byte array mapped to a String. The method calling this + * should encode the serialized object with `snowplowRawEventBytes.map(_.toChar)`. + * Reference: http://stackoverflow.com/questions/5250324/ + * @return either a set of validation errors or an Option-boxed CanonicalInput object, wrapped in + * a Scalaz ValidatioNel. */ private def convertOldSchema(line: Array[Byte]): ValidatedMaybeCollectorPayload = { - var snowplowRawEvent = new SnowplowRawEvent this.synchronized { thriftDeserializer.deserialize( diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/TsvLoader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/TsvLoader.scala index f4b10cffb..63d000782 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/TsvLoader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/TsvLoader.scala @@ -16,17 +16,14 @@ package loaders import scalaz._ import Scalaz._ -/** - * Loader for TSVs - */ -case class TsvLoader(adapter: String) extends Loader[String] { +/** Loader for TSVs */ +final case class TsvLoader(adapter: String) extends Loader[String] { /** * Converts the source TSV into a ValidatedMaybeCollectorPayload. - * * @param line A TSV - * @return either a set of validation errors or an Option-boxed - * CanonicalInput object, wrapped in a Scalaz ValidationNel. + * @return either a set of validation errors or an Option-boxed CanonicalInput object, wrapped in + * a Scalaz ValidationNel. */ def toCollectorPayload(line: String): ValidatedMaybeCollectorPayload = // Throw away the first two lines of Cloudfront web distribution access logs diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/collectorPayload.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/collectorPayload.scala index fcefeb243..cd84175f1 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/collectorPayload.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/collectorPayload.scala @@ -20,8 +20,7 @@ import Scalaz._ object CollectorPayload { /** - * A constructor version to use. Supports legacy - * tp1 (where no API vendor or version provided + * A constructor version to use. Supports legacy tp1 (where no API vendor or version provided * as well as Snowplow). */ def apply( @@ -37,8 +36,8 @@ object CollectorPayload { contextUserId: Option[String], api: CollectorApi, contentType: Option[String], - body: Option[String]): CollectorPayload = { - + body: Option[String] + ): CollectorPayload = { val source = CollectorSource(sourceName, sourceEncoding, sourceHostname) val context = CollectorContext( contextTimestamp, @@ -47,47 +46,37 @@ object CollectorPayload { contextRefererUri, contextHeaders, contextUserId) - CollectorPayload(api, querystring, contentType, body, source, context) } } object CollectorApi { - // Defaults for the tracker vendor and version - // before we implemented this into Snowplow. + // Defaults for the tracker vendor and version before we implemented this into Snowplow. // TODO: make private once the ThriftLoader is updated val SnowplowTp1 = CollectorApi("com.snowplowanalytics.snowplow", "tp1") - // To extract the API vendor and version from the - // the path to the requested object. - // TODO: move this to somewhere not specific to - // this collector + // To extract the API vendor and version from the the path to the requested object. + // TODO: move this to somewhere not specific to this collector private val ApiPathRegex = """^[\/]?([^\/]+)\/([^\/]+)[\/]?$""".r /** - * Parses the requested URI path to determine the - * specific API version this payload follows. - * + * Parses the requested URI path to determine the specific API version this payload follows. * @param path The request path - * @return a Validation boxing either a - * CollectorApi or a Failure String. + * @return a Validation boxing either a CollectorApi or a Failure String. */ def parse(path: String): Validation[String, CollectorApi] = path match { case ApiPathRegex(vnd, ver) => CollectorApi(vnd, ver).success case _ if isIceRequest(path) => SnowplowTp1.success case _ => - s"Request path ${path} does not match (/)vendor/version(/) pattern nor is a legacy /i(ce.png) request".fail + (s"Request path $path does not match (/)vendor/version(/) pattern nor is a " + + "legacy /i(ce.png) request").fail } /** - * Checks whether a request to - * a collector is a tracker - * hitting the ice pixel. - * + * Checks whether a request to a collector is a tracker hitting the ice pixel. * @param path The request path - * @return true if this is a request - * for the ice pixel + * @return true if this is a request for the ice pixel */ protected[loaders] def isIceRequest(path: String): Boolean = path.startsWith("/ice.png") || // Legacy name for /i @@ -96,8 +85,7 @@ object CollectorApi { } /** - * Unambiguously identifies the collector - * source of this input line. + * Unambiguously identifies the collector source of this input line. */ final case class CollectorSource( name: String, @@ -105,9 +93,7 @@ final case class CollectorSource( hostname: Option[String] ) -/** - * Context derived by the collector. - */ +/** Context derived by the collector. */ final case class CollectorContext( timestamp: Option[DateTime], // Must have a timestamp ipAddress: Option[String], @@ -117,26 +103,16 @@ final case class CollectorContext( userId: Option[String] // User ID generated by collector-set third-party cookie ) -/** - * Define the vendor and version - * of the payload. - */ -final case class CollectorApi( - vendor: String, - version: String -) +/** Define the vendor and version of the payload. */ +final case class CollectorApi(vendor: String, version: String) /** - * The canonical input format for the ETL - * process: it should be possible to - * convert any collector input format to - * this format, ready for the main, - * collector-agnostic stage of the ETL. + * The canonical input format for the ETL process: it should be possible to convert any collector + * input format to this format, ready for the main, collector-agnostic stage of the ETL. */ final case class CollectorPayload( api: CollectorApi, querystring: List[NameValuePair], // Could be empty in future trackers - contentType: Option[String], // Not always set body: Option[String], // Not set for GETs source: CollectorSource, diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/BadRow.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/BadRow.scala index f91862989..9fc4a0b00 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/BadRow.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/BadRow.scala @@ -14,22 +14,19 @@ package com.snowplowanalytics.snowplow.enrich.common.outputs import com.snowplowanalytics.iglu.client.ProcessingMessageNel import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ +import io.circe.jackson._ +import io.circe.syntax._ import org.joda.time.{DateTime, DateTimeZone} import org.joda.time.format.DateTimeFormat import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ -/** - * Alternate BadRow constructors - */ +/** Alternate BadRow constructors */ object BadRow { /** * Constructor using Strings instead of ProcessingMessages - * * @param line * @param errors * @return a BadRow @@ -39,59 +36,61 @@ object BadRow { /** * For rows which are so too long to send to Kinesis and so cannot contain their own original line - * * @param line * @param errors * @param tstamp * @return a BadRow */ - def oversizedRow(size: Long, errors: NonEmptyList[String], tstamp: Long = System.currentTimeMillis()): String = - compact( - ("size" -> size) ~ - ("errors" -> errors.toList.map(e => fromJsonNode(e.toProcessingMessage.asJson))) ~ - ("failure_tstamp" -> tstamp) - ) + def oversizedRow( + size: Long, + errors: NonEmptyList[String], + tstamp: Long = System.currentTimeMillis() + ): String = + Json + .obj( + "size" := Json.fromLong(size), + "errors" := Json.arr( + errors.toList.map(e => jacksonToCirce(e.toProcessingMessage.asJson)): _*), + "failure_tstamp" := Json.fromLong(tstamp) + ) + .noSpaces } /** * Models our report on a bad row. Consists of: - * 1. Our original input line (which was meant - * to be a Snowplow enriched event) + * 1. Our original input line (which was meant to be a Snowplow enriched event) * 2. A non-empty list of our Validation errors * 3. A timestamp */ -case class BadRow( +final case class BadRow( val line: String, val errors: ProcessingMessageNel, val tstamp: Long = System.currentTimeMillis() ) { // An ISO valid timestamp formatter - private val TstampFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(DateTimeZone.UTC) + private val TstampFormat = + DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(DateTimeZone.UTC) /** - * Converts a TypeHierarchy into a JSON containing - * each element. - * - * @return the TypeHierarchy as a json4s JValue + * Converts a TypeHierarchy into a JSON containing each element. + * @return the TypeHierarchy as a Json */ - def toJValue: JValue = - ("line" -> line) ~ - ("errors" -> errors.toList.map(e => fromJsonNode(e.asJson))) ~ - ("failure_tstamp" -> this.getTimestamp(tstamp)) + def toJson: Json = + Json.obj( + "line" := Json.fromString(line), + "errors" := Json.arr(errors.toList.map(e => jacksonToCirce(e.asJson)): _*), + "failure_tstamp" := getTimestamp(tstamp) + ) /** - * Converts our BadRow into a single JSON encapsulating - * both the input line and errors. - * + * Converts our BadRow into a single JSON encapsulating both the input line and errors. * @return this BadRow as a compact stringified JSON */ - def toCompactJson: String = - compact(this.toJValue) + def toCompactJson: String = toJson.noSpaces /** * Returns an ISO valid timestamp - * * @param tstamp The Timestamp to convert * @return the formatted Timestamp */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala index b5ec1df5b..4504404ce 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala @@ -34,19 +34,12 @@ import org.apache.http.client.utils.URLEncodedUtils import scalaz._ import Scalaz._ -/** - * General-purpose utils to help the - * ETL process along. - */ +/** General-purpose utils to help the ETL process along. */ object ConversionUtils { - private val UrlSafeBase64 = new Base64(true) // true means "url safe" - /** - * Simple case class wrapper around the - * components of a URI. - */ - case class UriComponents( + /** Simple case class wrapper around the components of a URI. */ + final case class UriComponents( // Required scheme: String, host: String, @@ -57,18 +50,11 @@ object ConversionUtils { fragment: Option[String]) /** - * Explodes a URI into its 6 components - * pieces. Simple code but we use it - * in multiple places - * - * @param uri The URI to explode into its - * constituent pieces - * - * @return The 6 components in a UriComponents - * case class + * Explodes a URI into its 6 components pieces. Simple code but we use it in multiple places + * @param uri The URI to explode into its constituent pieces + * @return The 6 components in a UriComponents case class */ def explodeUri(uri: URI): UriComponents = { - val port = uri.getPort // TODO: should we be using decodeString below instead? @@ -94,10 +80,8 @@ object ConversionUtils { } /** - * Quick helper to make sure our Strings are TSV-safe, - * i.e. don't include tabs, special characters, newlines - * etc. - * + * Quick helper to make sure our Strings are TSV-safe, i.e. don't include tabs, special + * characters, newlines, etc. * @param str The string we want to make safe * @return a safe String */ @@ -105,14 +89,9 @@ object ConversionUtils { fixTabsNewlines(str).orNull /** - * Replaces tabs with four spaces and removes - * newlines altogether. - * - * Useful to prepare user-created strings for - * fragile storage formats like TSV. - * + * Replaces tabs with four spaces and removes newlines altogether. + * Useful to prepare user-created strings for fragile storage formats like TSV. * @param str The String to fix - * @return The String with tabs and newlines fixed. */ def fixTabsNewlines(str: String): Option[String] = { @@ -125,21 +104,15 @@ object ConversionUtils { /** * Decodes a URL-safe Base64 string. - * - * For details on the Base 64 Encoding with URL - * and Filename Safe Alphabet see: - * + * For details on the Base 64 Encoding with URL and Filename Safe Alphabet see: * http://tools.ietf.org/html/rfc4648#page-7 - * - * @param str The encoded string to be - * decoded + * @param str The encoded string to be decoded * @param field The name of the field * @return a Scalaz Validation, wrapping either an * an error String or the decoded String */ // TODO: probably better to change the functionality and signature // a little: - // // 1. Signature -> : Validation[String, Option[String]] // 2. Functionality: // 1. If passed in null or "", return Success(None) @@ -151,17 +124,15 @@ object ConversionUtils { result.success } catch { case NonFatal(e) => - "Field [%s]: exception Base64-decoding [%s] (URL-safe encoding): [%s]".format(field, str, e.getMessage).fail + "Field [%s]: exception Base64-decoding [%s] (URL-safe encoding): [%s]" + .format(field, str, e.getMessage) + .fail } /** * Encodes a URL-safe Base64 string. - * - * For details on the Base 64 Encoding with URL - * and Filename Safe Alphabet see: - * + * For details on the Base 64 Encoding with URL and Filename Safe Alphabet see: * http://tools.ietf.org/html/rfc4648#page-7 - * * @param str The string to be encoded * @return the string encoded in URL-safe Base64 */ @@ -172,15 +143,12 @@ object ConversionUtils { /** * Validates that the given field contains a valid UUID. - * * @param field The name of the field being validated * @param str The String hopefully containing a UUID - * @return a Scalaz ValidatedString containing either - * the original String on Success, or an error - * String on Failure. + * @return a Scalaz ValidatedString containing either the original String on Success, or an error + * String on Failure. */ val validateUuid: (String, String) => ValidatedString = (field, str) => { - def check(s: String)(u: UUID): Boolean = (u != null && s.toLowerCase == u.toString) val uuid = Try(UUID.fromString(str)).toOption.filter(check(str)) uuid match { @@ -192,41 +160,32 @@ object ConversionUtils { /** * @param field The name of the field being validated * @param str The String hopefully parseable as an integer - * @return a Scalaz ValidatedString containing either - * the original String on Success, or an error - * String on Failure. + * @return a Scalaz ValidatedString containing either the original String on Success, or an error + * String on Failure. */ val validateInteger: (String, String) => ValidatedString = (field, str) => { try { str.toInt str.success } catch { - case _: java.lang.NumberFormatException => s"Field [$field]: [$str] is not a valid integer".fail + case _: java.lang.NumberFormatException => + s"Field [$field]: [$str] is not a valid integer".fail } } /** - * Decodes a String in the specific encoding, - * also removing: + * Decodes a String in the specific encoding, also removing: * * Newlines - because they will break Hive * * Tabs - because they will break non-Hive * targets (e.g. Infobright) - * - * IMPLDIFF: note that this version, unlike - * the Hive serde version, does not call - * cleanUri. This is because we cannot assume - * that str is a URI which needs 'cleaning'. - * - * TODO: simplify this when we move to a more - * robust output format (e.g. Avro) - as then + * IMPLDIFF: note that this version, unlike the Hive serde version, does not call + * cleanUri. This is because we cannot assume that str is a URI which needs 'cleaning'. + * TODO: simplify this when we move to a more robust output format (e.g. Avro) - as then * no need to remove line breaks, tabs etc - * * @param enc The encoding of the String * @param field The name of the field * @param str The String to decode - * - * @return a Scalaz Validation, wrapping either - * an error String or the decoded String + * @return a Scalaz Validation, wrapping either an error String or the decoded String */ val decodeString: (Charset, String, String) => ValidatedString = (enc, field, str) => try { @@ -238,40 +197,29 @@ object ConversionUtils { r.success } catch { case NonFatal(e) => - "Field [%s]: Exception URL-decoding [%s] (encoding [%s]): [%s]".format(field, str, enc, e.getMessage).fail - } + "Field [%s]: Exception URL-decoding [%s] (encoding [%s]): [%s]" + .format(field, str, enc, e.getMessage) + .fail + } /** - * On 17th August 2013, Amazon made an - * unannounced change to their CloudFront - * log format - they went from always encoding - * % characters, to only encoding % characters - * which were not previously encoded. For a - * full discussion of this see: - * + * On 17th August 2013, Amazon made an unannounced change to their CloudFront + * log format - they went from always encoding % characters, to only encoding % characters + * which were not previously encoded. For a full discussion of this see: * https://forums.aws.amazon.com/thread.jspa?threadID=134017&tstart=0# - * - * On 14th September 2013, Amazon rolled out a further fix, - * from which point onwards all fields, including the - * referer and useragent, would have %s double-encoded. - * - * This causes issues, because the ETL process expects - * referers and useragents to be only single-encoded. - * - * This function turns a double-encoded percent (%) into - * a single-encoded one. - * + * On 14th September 2013, Amazon rolled out a further fix, from which point onwards all fields, + * including the referer and useragent, would have %s double-encoded. + * This causes issues, because the ETL process expects referers and useragents to be only + * single-encoded. + * This function turns a double-encoded percent (%) into a single-encoded one. * Examples: * 1. "page=Celestial%25Tarot" - no change (only single encoded) * 2. "page=Dreaming%2520Way%2520Tarot" -> "page=Dreaming%20Way%20Tarot" * 3. "loading 30%2525 complete" -> "loading 30%25 complete" - * * Limitation of this approach: %2588 is ambiguous. Is it a: * a) A double-escaped caret "ˆ" (%2588 -> %88 -> ^), or: * b) A single-escaped "%88" (%2588 -> %88) - * * This code assumes it's a). - * * @param str The String which potentially has double-encoded %s * @return the String with %s now single-encoded */ @@ -280,19 +228,15 @@ object ConversionUtils { /** * Decode double-encoded percents, then percent decode - * * @param field The name of the field * @param str The String to decode - * - * @return a Scalaz Validation, wrapping either - * an error String or the decoded String + * @return a Scalaz Validation, wrapping either an error String or the decoded String */ def doubleDecode(field: String, str: String): ValidatedString = ConversionUtils.decodeString(UTF_8, field, singleEncodePcts(str)) /** * Encodes a string in the specified encoding - * * @param enc The encoding to be used * @param str The string which needs to be URLEncoded * @return a URL encoded string @@ -303,11 +247,10 @@ object ConversionUtils { /** * Parses a string to create a [[URI]]. * Parsing is relaxed, i.e. even if a URL is not correctly percent-encoded or not RFC 3986-compliant, it can be parsed. - * * @param uri String containing the URI to parse. * @return [[Validation]] wrapping the result of the parsing: - * - [[Success]] with the parsed URI if there was no error or with [[None]] if the input was `null`. - * - [[Failure]] with the error message if something went wrong. + * - [[Success]] with the parsed URI if there was no error or with [[None]] if the input was `null`. + * - [[Failure]] with the error message if something went wrong. */ def stringToUri(uri: String): Validation[String, Option[URI]] = Try( @@ -319,7 +262,9 @@ object ConversionUtils { parsed.success case util.Failure(javaErr) => implicit val c = - UriConfig(decoder = PercentDecoder(ignoreInvalidPercentEncoding = true), encoder = percentEncode -- '+') + UriConfig( + decoder = PercentDecoder(ignoreInvalidPercentEncoding = true), + encoder = percentEncode -- '+') Uri .parseTry(uri) .map(_.toJavaURI) match { @@ -334,7 +279,6 @@ object ConversionUtils { /** * Attempt to extract the querystring from a URI as a map - * * @param uri URI containing the querystring * @param encoding Encoding of the URI */ @@ -348,20 +292,10 @@ object ConversionUtils { } /** - * Extract a Scala Int from - * a String, or error. - * - * @param str The String - * which we hope is an - * Int - * @param field The name of the - * field we are trying to - * process. To use in our - * error message - * @return a Scalaz Validation, - * being either a - * Failure String or - * a Success JInt + * Extract a Scala Int from a String, or error. + * @param str The String which we hope is an Int + * @param field The name of the field we are trying to process. To use in our error message + * @return a Scalaz Validation, being either a Failure String or a Success JInt */ val stringToJInteger: (String, String) => Validation[String, JInteger] = (field, str) => if (Option(str).isEmpty) { @@ -377,27 +311,18 @@ object ConversionUtils { } /** - * Convert a String to a String containing a - * Redshift-compatible Double. - * - * Necessary because Redshift does not support all - * Java Double syntaxes e.g. "3.4028235E38" - * - * Note that this code does NOT check that the - * value will fit within a Redshift Double - - * meaning Redshift may silently round this number - * on load. - * - * @param str The String which we hope contains - * a Double - * @param field The name of the field we are - * validating. To use in our error message - * @return a Scalaz Validation, being either - * a Failure String or a Success String + * Convert a String to a String containing a Redshift-compatible Double. + * Necessary because Redshift does not support all Java Double syntaxes e.g. "3.4028235E38" + * Note that this code does NOT check that the value will fit within a Redshift Double - + * meaning Redshift may silently round this number on load. + * @param str The String which we hope contains a Double + * @param field The name of the field we are validating. To use in our error message + * @return a Scalaz Validation, being either a Failure String or a Success String */ val stringToDoublelike: (String, String) => ValidatedString = (field, str) => try { - if (Option(str).isEmpty || str == "null") { // "null" String check is LEGACY to handle a bug in the JavaScript tracker + if (Option(str).isEmpty || str == "null") { + // "null" String check is LEGACY to handle a bug in the JavaScript tracker null.asInstanceOf[String].success } else { val jbigdec = new JBigDecimal(str) @@ -410,17 +335,14 @@ object ConversionUtils { /** * Convert a String to a Double - * - * @param str The String which we hope contains - * a Double - * @param field The name of the field we are - * validating. To use in our error message - * @return a Scalaz Validation, being either - * a Failure String or a Success Double + * @param str The String which we hope contains a Double + * @param field The name of the field we are validating. To use in our error message + * @return a Scalaz Validation, being either a Failure String or a Success Double */ def stringToMaybeDouble(field: String, str: String): Validation[String, Option[Double]] = try { - if (Option(str).isEmpty || str == "null") { // "null" String check is LEGACY to handle a bug in the JavaScript tracker + if (Option(str).isEmpty || str == "null") { + // "null" String check is LEGACY to handle a bug in the JavaScript tracker None.success } else { val jbigdec = new JBigDecimal(str) @@ -457,20 +379,10 @@ object ConversionUtils { } /** - * Extract a Java Byte representing - * 1 or 0 only from a String, or error. - * - * @param str The String - * which we hope is an - * Byte - * @param field The name of the - * field we are trying to - * process. To use in our - * error message - * @return a Scalaz Validation, - * being either a - * Failure String or - * a Success Byte + * Extract a Java Byte representing 1 or 0 only from a String, or error. + * @param str The String which we hope is an Byte + * @param field The name of the field we are trying to process. To use in our error message + * @return a Scalaz Validation, being either a Failure String or a Success Byte */ val stringToBooleanlikeJByte: (String, String) => Validation[String, JByte] = (field, str) => str match { @@ -480,14 +392,10 @@ object ConversionUtils { } /** - * Converts a String of value "1" or "0" - * to true or false respectively. - * + * Converts a String of value "1" or "0" to true or false respectively. * @param str The String to convert - * @return True for "1", false for "0", or - * an error message for any other - * value, all boxed in a Scalaz - * Validation + * @return True for "1", false for "0", or an error message for any other value, all boxed in a + * Scalaz Validation */ val stringToBoolean: (String, String) => Validation[String, Boolean] = (field, str) => if (str == "1") { @@ -499,12 +407,9 @@ object ConversionUtils { } /** - * Truncates a String - useful for making sure - * Strings can't overflow a database field. - * + * Truncates a String - useful for making sure Strings can't overflow a database field. * @param str The String to truncate - * @param length The maximum length of the String - * to keep + * @param length The maximum length of the String to keep * @return the truncated String */ def truncate(str: String, length: Int): String = @@ -515,9 +420,7 @@ object ConversionUtils { } /** - * Helper to convert a Boolean value to a Byte. - * Does not require any validation. - * + * Helper to convert a Boolean value to a Byte. Does not require any validation. * @param bool The Boolean to convert into a Byte * @return 0 if false, 1 if true */ @@ -525,15 +428,10 @@ object ConversionUtils { (if (bool) 1 else 0).toByte /** - * Helper to convert a Byte value - * (1 or 0) into a Boolean. - * - * @param b The Byte to turn - * into a Boolean - * @return the Boolean value of b, or - * an error message if b is - * not 0 or 1 - all boxed in a - * Scalaz Validation + * Helper to convert a Byte value (1 or 0) into a Boolean. + * @param b The Byte to turn into a Boolean + * @return the Boolean value of b, or an error message if b is not 0 or 1 - all boxed in a + * Scalaz Validation */ def byteToBoolean(b: Byte): Validation[String, Boolean] = if (b == 0) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala index fc1467c48..db1514db9 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala @@ -22,11 +22,10 @@ object HttpClient { // The defaults are from scalaj library val DEFAULT_CONNECTION_TIMEOUT_MS = 1000 - val DEFAULT_READ_TIMEOUT_MS = 5000 + val DEFAULT_READ_TIMEOUT_MS = 5000 /** * Blocking method to get body of HTTP response - * * @param request assembled request object * @return validated body of HTTP request */ @@ -41,7 +40,6 @@ object HttpClient { /** * Build HTTP request object - * * @param uri full URI to request * @param authUser optional username for basic auth * @param authPassword optional password for basic auth @@ -67,18 +65,21 @@ object HttpClient { implicit class RichHttpRequest(request: HttpRequest) { def maybeAuth(user: Option[String], password: Option[String]): HttpRequest = - if (user.isDefined || password.isDefined) request.auth(user.getOrElse(""), password.getOrElse("")) + if (user.isDefined || password.isDefined) + request.auth(user.getOrElse(""), password.getOrElse("")) else request def maybeTimeout(connectionTimeout: Option[Long], readTimeout: Option[Long]): HttpRequest = (connectionTimeout, readTimeout) match { case (Some(ct), Some(rt)) => request.timeout(ct.toInt, rt.toInt) - case (Some(ct), None) => request.timeout(ct.toInt, DEFAULT_READ_TIMEOUT_MS) - case (None, Some(rt)) => request.timeout(DEFAULT_CONNECTION_TIMEOUT_MS, rt.toInt) - case _ => request.timeout(DEFAULT_CONNECTION_TIMEOUT_MS, DEFAULT_READ_TIMEOUT_MS) + case (Some(ct), None) => request.timeout(ct.toInt, DEFAULT_READ_TIMEOUT_MS) + case (None, Some(rt)) => request.timeout(DEFAULT_CONNECTION_TIMEOUT_MS, rt.toInt) + case _ => request.timeout(DEFAULT_CONNECTION_TIMEOUT_MS, DEFAULT_READ_TIMEOUT_MS) } def maybePostData(body: Option[String]): HttpRequest = - body.map(data => request.postData(data).header("content-type", "application/json")).getOrElse(request) + body + .map(data => request.postData(data).header("content-type", "application/json")) + .getOrElse(request) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala index 0623e497f..72ade02e1 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala @@ -13,61 +13,68 @@ package com.snowplowanalytics.snowplow.enrich.common.utils import io.gatling.jsonpath.{JsonPath => GatlingJsonPath} +import io.circe._ import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods.mapper -/** - * Wrapper for `io.gatling.jsonpath` for `json4s` and `scalaz` - */ +/** Wrapper for `io.gatling.jsonpath` for circe and scalaz */ object JsonPath { - private val json4sMapper = mapper - /** - * Wrapper method for not throwing an exception on JNothing, - * representing it as invalid JSON - * + * Wrapper method for not throwing an exception on JNothing, representing it as invalid JSON * @param json JSON value, possibly JNothing * @return successful POJO on any JSON except JNothing */ - def convertToJValue(json: JValue): Validation[String, Object] = - json match { - case JNothing => "JSONPath error: Nothing was given".failure - case other => json4sMapper.convertValue(other, classOf[Object]).success - } + def convertToJson(json: Json): Object = + io.circe.jackson.mapper.convertValue(json, classOf[Object]) + + def testMe(): Unit = { + import io.circe.literal._ + val json1 = json"""{"latitude": 43.1, "longitude": 32.1}""" + val json2 = Json.obj( + ("latitude", Json.fromDoubleOrNull(43.1)), + ("longitude", Json.fromDoubleOrNull(32.1)) + ) + import cats.syntax.eq._ + println(json1 === json2) + val pojo1 = io.circe.jackson.mapper.convertValue(json1, classOf[Object]) + val pojo2 = io.circe.jackson.mapper.convertValue(json2, classOf[Object]) + println(pojo1 == pojo2) + import cats.syntax.either._ + println( + Either.catchNonFatal( + pojo1.asInstanceOf[java.util.HashMap[Object, Object]].get("latitude").getClass)) + println( + Either.catchNonFatal( + pojo2.asInstanceOf[java.util.HashMap[Object, Object]].get("latitude").getClass)) + } /** * Pimp-up JsonPath class to work with JValue * Unlike `query(jsonPath, json)` it gives empty list on any error (like JNothing) - * * @param jsonPath precompiled with [[compileQuery]] JsonPath object */ - implicit class Json4sExtractor(jsonPath: GatlingJsonPath) { - def json4sQuery(json: JValue): List[JValue] = - convertToJValue(json) match { - case Success(pojo) => jsonPath.query(pojo).map(anyToJValue).toList - case Failure(_) => Nil - } + implicit class CirceExtractor(jsonPath: GatlingJsonPath) { + def circeQuery(json: Json): List[Json] = { + val pojo = convertToJson(json) + jsonPath.query(pojo).map(anyToJson).toList + } } /** - * Query some JSON by `jsonPath` - * It always return List, even for single match - * Unlike `jValue.json4sQuery(stringPath)` it gives error if JNothing was given + * Query some JSON by `jsonPath`. It always return List, even for single match. + * Unlike `json.circeQuery(stringPath)` it gives error if JNothing was given */ - def query(jsonPath: String, json: JValue): Validation[String, List[JValue]] = - convertToJValue(json).flatMap { pojo => - GatlingJsonPath.query(jsonPath, pojo) match { - case Right(iterator) => iterator.map(anyToJValue).toList.success - case Left(error) => error.reason.fail - } + def query(jsonPath: String, json: Json): Validation[String, List[Json]] = { + val pojo = convertToJson(json) + GatlingJsonPath.query(jsonPath, pojo) match { + case Right(iterator) => iterator.map(anyToJson).toList.success + case Left(error) => error.reason.fail } + } /** * Precompile JsonPath query - * * @param query JsonPath query as a string * @return valid JsonPath object either error message */ @@ -77,24 +84,21 @@ object JsonPath { /** * Wrap list of values into JSON array if several values present * Use in conjunction with `query`. JNothing will represent absent value - * * @param values list of JSON values * @return array if there's >1 values in list */ - def wrapArray(values: List[JValue]): JValue = values match { - case Nil => JNothing + def wrapArray(values: List[Json]): Json = values match { + case Nil => Json.fromValues(List.empty) case one :: Nil => one - case many => JArray(many) + case many => Json.fromValues(many) } /** * Convert POJO to JValue with `jackson` mapper - * * @param any raw JVM type representing JSON - * @return JValue + * @return Json */ - private def anyToJValue(any: Any): JValue = - if (any == null) JNull - else json4sMapper.convertValue(any, classOf[JValue]) - + private def anyToJson(any: Any): Json = + if (any == null) Json.Null + else io.circe.jackson.mapper.convertValue(any, classOf[Json]) } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala index 9e5d87125..2d8b83870 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala @@ -14,20 +14,16 @@ package com.snowplowanalytics.snowplow.enrich.common.utils import java.math.{BigInteger => JBigInteger} -import scala.util.control.NonFatal - +import cats.syntax.either._ import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import io.circe._ +import io.circe.jackson._ import org.joda.time.{DateTime, DateTimeZone} import org.joda.time.format.{DateTimeFormat, DateTimeFormatter} import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ -/** - * Contains general purpose extractors and other - * utilities for JSONs. Jackson-based. - */ +/** Contains general purpose extractors and other utilities for JSONs. Jackson-based. */ object JsonUtils { type DateTimeFields = Option[Tuple2[NonEmptyList[String], DateTimeFormatter]] @@ -38,83 +34,56 @@ object JsonUtils { private val JsonSchemaDateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(DateTimeZone.UTC) - /** - * Validates a String as correct JSON. - */ + /** Validates a String as correct JSON. */ val extractUnencJson: (String, String) => Validation[String, String] = validateAndReformatJson - /** - * Decodes a Base64 (URL safe)-encoded String then - * validates it as correct JSON. - */ + /** Decodes a Base64 (URL safe)-encoded String then validates it as correct JSON. */ val extractBase64EncJson: (String, String) => Validation[String, String] = (field, str) => ConversionUtils .decodeBase64Url(field, str) .flatMap(json => validateAndReformatJson(field, json)) /** - * Converts a Joda DateTime into - * a JSON Schema-compatible date-time string. - * - * @param dateTime The Joda DateTime - * to convert to a timestamp String + * Converts a Joda DateTime into a JSON Schema-compatible date-time string. + * @param dateTime The Joda DateTime to convert to a timestamp String * @return the timestamp String */ - private[utils] def toJsonSchemaDateTime(dateTime: DateTime): String = JsonSchemaDateTimeFormat.print(dateTime) + private[utils] def toJsonSchemaDateTime(dateTime: DateTime): String = + JsonSchemaDateTimeFormat.print(dateTime) /** - * Converts a boolean-like String of value "true" - * or "false" to a JBool value of true or false - * respectively. Any other value becomes a - * JString. - * - * No erroring if the String is not boolean-like, - * leave it to eventual JSON Schema validation + * Converts a boolean-like String of value "true" or "false" to a JBool value of true or false + * respectively. Any other value becomes a JString. + * No erroring if the String is not boolean-like, leave it to eventual JSON Schema validation * to enforce that. - * * @param str The boolean-like String to convert - * @return true for "true", false for "false", - * and otherwise a JString wrapping the - * original String + * @return true, false, and otherwise a JString wrapping the original String */ - private[utils] def booleanToJValue(str: String): JValue = str match { - case "true" => JBool(true) - case "false" => JBool(false) - case _ => JString(str) + private[utils] def booleanToJson(str: String): Json = str match { + case "true" => Json.True + case "false" => Json.False + case _ => Json.fromString(str) } /** - * Converts an integer-like String to a - * JInt value. Any other value becomes a - * JString. - * - * No erroring if the String is not integer-like, - * leave it to eventual JSON Schema validation + * Converts an integer-like String to a JInt value. Any other value becomes a JString. + * No erroring if the String is not integer-like, leave it to eventual JSON Schema validation * to enforce that. - * * @param str The integer-like String to convert - * @return a JInt if the String was integer-like, - * or else a JString wrapping the original - * String. + * @return a JInt if the String was integer-like, otherwise a JString wrapping the original. */ - private[utils] def integerToJValue(str: String): JValue = - try { - JInt(new JBigInteger(str)) - } catch { - case nfe: NumberFormatException => - JString(str) + private[utils] def integerToJson(str: String): Json = + Either.catchNonFatal(new JBigInteger(str)) match { + case Right(bigInt) => Json.fromBigInt(bigInt) + case _ => Json.fromString(str) } /** - * Reformats a non-standard date-time into a format - * compatible with JSON Schema's date-time format - * validation. If the String does not match the - * expected date format, then return the original String. - * - * @param str The date-time-like String to reformat - * to pass JSON Schema validation - * @return the reformatted date-time String if - * possible, or otherwise the original String + * Reformats a non-standard date-time into a format compatible with JSON Schema's date-time + * format validation. If the String does not match the expected date format, then return the + * original String. + * @param str The date-time-like String to reformat to pass JSON Schema validation + * @return the reformatted date-time String if possible, or otherwise the original String */ def toJsonSchemaDateTime(str: String, fromFormat: DateTimeFormatter): String = try { @@ -125,38 +94,31 @@ object JsonUtils { } /** - * Converts an incoming key, value into a json4s JValue. - * Uses the lists of keys which should contain bools, - * ints and dates to apply specific processing to - * those values when found. - * - * @param key The key of the field to generate. Also used - * to determine what additional processing should - * be applied to the value + * Converts an incoming key, value into a Json. Uses the lists of keys which should + * contain bools, ints and dates to apply specific processing to those values when found. + * @param key The key of the field to generate. Also used to determine what additional + * processing should be applied to the value * @param value The value of the field - * @param bools A List of keys whose values should be - * processed as boolean-like Strings - * @param ints A List of keys whose values should be - * processed as integer-like Strings - * @param dates If Some, a NEL of keys whose values should - * be treated as date-time-like Strings, which will - * require processing from the specified format - * @return a JField, containing the original key and the - * processed String, now as a JValue + * @param bools A List of keys whose values should be processed as boolean-like Strings + * @param ints A List of keys whose values should be processed as integer-like Strings + * @param dates If Some, a NEL of keys whose values should be treated as date-time-like Strings, + * which will require processing from the specified format + * @return a JField, containing the original key and the processed String, now as a JValue */ - def toJField( + def toJson( key: String, value: String, bools: List[String], ints: List[String], - dateTimes: DateTimeFields): JField = { - + dateTimes: DateTimeFields + ): (String, Json) = { val v = (value, dateTimes) match { - case ("", _) => JNull - case _ if bools.contains(key) => booleanToJValue(value) - case _ if ints.contains(key) => integerToJValue(value) - case (_, Some((nel, fmt))) if nel.toList.contains(key) => JString(toJsonSchemaDateTime(value, fmt)) - case _ => JString(value) + case ("", _) => Json.Null + case _ if bools.contains(key) => booleanToJson(value) + case _ if ints.contains(key) => integerToJson(value) + case (_, Some((nel, fmt))) if nel.toList.contains(key) => + Json.fromString(toJsonSchemaDateTime(value, fmt)) + case _ => Json.fromString(value) } (key, v) } @@ -165,40 +127,36 @@ object JsonUtils { * Validates and reformats a JSON: * 1. Checks the JSON is valid * 2. Reformats, including removing unnecessary whitespace - * * @param field the name of the field containing the JSON * @param str the String hopefully containing JSON - * @return a Scalaz Validation, wrapping either an error - * String or the reformatted JSON String + * @return a Scalaz Validation, wrapping either an error String or the reformatted JSON String */ - private[utils] def validateAndReformatJson(field: String, str: String): Validation[String, String] = - extractJson(field, str).map(j => compact(fromJsonNode(j))) + private[utils] def validateAndReformatJson( + field: String, + str: String + ): Validation[String, String] = + extractJson(field, str).map(_.noSpaces) /** * Converts a JSON string into a Validation[String, JsonNode] * Version 2.6.7 of jackson can send back null instead of exception here - * * @param field The name of the field containing JSON * @param instance The JSON string to parse - * @return a Scalaz Validation, wrapping either an error - * String or the extracted JsonNode + * @return a Scalaz Validation, wrapping either an error String or the extracted JsonNode */ - def extractJson(field: String, instance: String): Validation[String, JsonNode] = - try { - Option(Mapper.readTree(instance)) - .toSuccess(s"Field [$field]: invalid JSON [$instance] with parsing error: mapping resulted in null") - } catch { - case NonFatal(e) => - s"Field [$field]: invalid JSON [$instance] with parsing error: ${stripInstanceEtc(e.getMessage).orNull}".fail + def extractJson(field: String, instance: String): Validation[String, Json] = + io.circe.parser.parse(instance) match { + case Right(js) => js.success + case Left(e) => + s"Field [$field]: invalid JSON [$instance] with parsing error: ${e.message}".fail } + def extractJsonNode(field: String, instance: String): Validation[String, JsonNode] = + extractJson(field, instance).map(circeToJackson) + /** * Converts a JSON string into a JsonNode. - * - * UNSAFE - only use it for Strings you have - * created yourself. Use extractJson for all - * external Strings. - * + * UNSAFE - only use it for Strings you have created yourself. Use extractJson otherwise. * @param instance The JSON string to parse * @return the extracted JsonNode */ @@ -208,18 +166,13 @@ object JsonUtils { /** * Strips the instance information from a Jackson * parsing exception message: - * * "... at [Source: java.io.StringReader@1fe7a8f8; line: 1, column: 2]"" * ^^^^^^^^ - * * Also removes any control characters and replaces * tabs with 4 spaces. - * - * @param message The exception message which needs - * tidying up - * @return the same exception message, but with - * instance information etc removed. Option-boxed - * because the message can be null + * @param message The exception message which needs tidying up + * @return the same exception message, but with instance information etc removed. Option-boxed + * because the message can be null */ def stripInstanceEtc(message: String): Option[String] = for (m <- Option(message)) yield { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala index a01b9e273..e4c1db14e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala @@ -64,7 +64,8 @@ object MapTransformer { type Value = String type Field = String - // A transformation takes a Key and Value and returns a Scalaz Validation with String for Failure and anything for Success + // A transformation takes a Key and Value and returns a Scalaz Validation with String for Failure + // and anything for Success type TransformFunc = Function2[Key, Value, Validation[String, _]] // Our source map @@ -77,36 +78,28 @@ object MapTransformer { type SettersMap = Map[Key, Method] /** - * A factory to generate a new object using - * a TransformMap. - * - * @param sourceMap Contains the source data to - * apply to the obj - * @param transformMap Determines how the source - * data should be transformed - * before storing in the obj - * @return a ValidationNel containing either a Nel - * of error Strings, or the new object + * A factory to generate a new object using a TransformMap. + * @param sourceMap Contains the source data to apply to the obj + * @param transformMap Determines how the data should be transformed before storing in the obj + * @return a ValidationNel containing either a Nel of error Strings, or the new object */ - def generate[T <: AnyRef](sourceMap: SourceMap, transformMap: TransformMap)(implicit m: Manifest[T]): Validated[T] = { + def generate[T <: AnyRef](sourceMap: SourceMap, transformMap: TransformMap)( + implicit m: Manifest[T]): Validated[T] = { val newInst = m.runtimeClass.newInstance() val result = _transform(newInst, sourceMap, transformMap, getSetters(m.runtimeClass)) - result.flatMap(s => newInst.asInstanceOf[T].success) // On success, replace the field count with the new instance + // On success, replace the field count with the new instance + result.flatMap(s => newInst.asInstanceOf[T].success) } /** - * An implicit conversion to take any Object and make it - * Transformable. - * + * An implicit conversion to take any Object and make it Transformable. * @param obj Any Object * @return the new Transformable class, with manifest attached */ - implicit def makeTransformable[T <: AnyRef](obj: T)(implicit m: Manifest[T]) = new TransformableClass[T](obj) + implicit def makeTransformable[T <: AnyRef](obj: T)(implicit m: Manifest[T]) = + new TransformableClass[T](obj) - /** - * A pimped object, now transformable by - * using the transform method. - */ + /** A pimped object, now transformable by using the transform method. */ class TransformableClass[T](obj: T)(implicit m: Manifest[T]) { // Do all the reflection for the setters we need: @@ -114,46 +107,30 @@ object MapTransformer { private lazy val setters = getSetters(m.runtimeClass) /** - * Update the object by applying the contents - * of a SourceMap to the object using a TransformMap. - * - * @param sourceMap Contains the source data to - * apply to the obj - * @param transformMap Determines how the source - * data should be transformed - * before storing in the obj - * @return a ValidationNel containing either a Nel - * of error Strings, or the count of - * updated fields + * Update the object by applying the contents of a SourceMap to the object using a TransformMap. + * @param sourceMap Contains the source data to apply to the obj + * @param transformMap Determines how the data should be transformed before storing in the obj + * @return a ValidationNel containing a Nel of error Strings, or the count of updated fields */ def transform(sourceMap: SourceMap, transformMap: TransformMap): ValidationNel[String, Int] = _transform[T](obj, sourceMap, transformMap, setters) } /** - * General-purpose method to update any object - * by applying the contents of a SourceMap to - * the object using a TransformMap. We use the - * SettersMap to update the object. - * + * General-purpose method to update any object by applying the contents of a SourceMap to + * the object using a TransformMap. We use the SettersMap to update the object. * @param obj Any Object - * @param sourceMap Contains the source data to - * apply to the obj - * @param transformMap Determines how the source - * data should be transformed - * before storing in the obj - * @param setters Provides access to the obj's - * setX() methods - * @return a ValidationNel containing either a Nel - * of error Strings, or the count of - * updated fields + * @param sourceMap Contains the source data to apply to the obj + * @param transformMap Determines how the data should be transformed before storing in the obj + * @param setters Provides access to the obj's setX() methods + * @return a ValidationNel containing a Nel of error Strings, or the count of updated fields */ private def _transform[T]( obj: T, sourceMap: SourceMap, transformMap: TransformMap, - setters: SettersMap): ValidationNel[String, Int] = { - + setters: SettersMap + ): ValidationNel[String, Int] = { val results: List[Validation[String, Int]] = sourceMap.map { case (key, in) => if (transformMap.contains(key)) { @@ -198,37 +175,30 @@ object MapTransformer { } /** - * Lowercases the first character in - * a String. - * - * @param s The String to lowercase the - * first letter of - * @return s with the first character - * in lowercase + * Lowercases the first character in a String. + * @param s The String to lowercase the first letter of + * @return s with the first character in lowercase */ private def lowerFirst(s: String): String = s.substring(0, 1).toLowerCase + s.substring(1) /** - * Gets the field name from a setter Method, - * by cutting out "set" and lowercasing the - * first character after in the setter's name. - * - * @param setter The Method from which we will - * reverse-engineer the field name + * Gets the field name from a setter Method, by cutting out "set" and lowercasing the first + * character after in the setter's name. + * @param setter The Method from which we will reverse-engineer the field name * @return the field name extracted from the setter */ private def setterToFieldName(setter: Method): String = lowerFirst(setter.getName.substring(3)) /** - * Gets all of the setter Methods - * from a manifest. - * - * @param c The manifest containing the - * setter methods to return + * Gets all of the setter Methods from a manifest. + * @param c The manifest containing the setter methods to return * @return the Map of setter Methods */ private def getSetters[T](c: Class[T]): SettersMap = - c.getDeclaredMethods.filter { _.getName.startsWith("set") }.groupBy { setterToFieldName(_) }.mapValues { _.head } + c.getDeclaredMethods + .filter { _.getName.startsWith("set") } + .groupBy { setterToFieldName(_) } + .mapValues { _.head } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazCirceUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazCirceUtils.scala new file mode 100644 index 000000000..b60ca4263 --- /dev/null +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazCirceUtils.scala @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2012-2019 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.enrich.common +package utils + +import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ +import io.circe._ +import scalaz._ +import Scalaz._ + +object ScalazCirceUtils { + + /** + * Returns a field of type A at the end of a JSON path + * @tparam A Type of the field to extract + * @param head The first field in the JSON path. Exists to ensure the path is nonempty + * @param tail The rest of the fields in the JSON path + * @return the list extracted from the JSON on success or an error String on failure + */ + def extract[A: Decoder: Manifest]( + config: Json, + head: String, + tail: String* + ): ValidatedMessage[A] = { + val path = head :: tail.toList + path.foldLeft(config.hcursor: ACursor) { case (c, f) => c.downField(f) }.as[A] match { + case Right(a) => a.success + case Left(e) => + val pathStr = path.mkString(".") + val clas = manifest[A] + s"Could not extract $pathStr as $clas from supplied JSON due to ${e.getMessage}".toProcessingMessage.fail + } + } +} diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazJson4sUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazJson4sUtils.scala deleted file mode 100644 index abf07b885..000000000 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazJson4sUtils.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012-2019 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.snowplow.enrich.common -package utils - -import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ -import org.json4s.{Formats, JNothing, JValue, MappingException} -import org.json4s.JsonDSL._ -import scalaz._ -import Scalaz._ - -object ScalazJson4sUtils { - - /** - * Returns a field of type A at the end of a - * JSON path - * - * @tparam A Type of the field to extract - * @param head The first field in the JSON path - * Exists to ensure the path is nonempty - * @param tail The rest of the fields in the - * JSON path - * @return the list extracted from the JSON on - * success or an error String on failure - */ - def extract[A: Manifest](config: JValue, head: String, tail: String*)( - implicit json4sFormats: Formats): ValidatedMessage[A] = { - val path = head +: tail - - // This check is necessary because attempting to follow - // an invalid path yields a JNothing, which would be - // interpreted as an empty list if type A is List[String] - if (fieldExists(config, head, tail: _*)) { - try { - path.foldLeft(config)(_ \ _).extract[A].success - } catch { - case me: MappingException => - s"Could not extract %s as %s from supplied JSON due to: ${me.getMessage}" - .format(path.mkString("."), manifest[A]) - .toProcessingMessage - .fail - } - } else s"JSON path %s not found".format(path.mkString(".")).toProcessingMessage.fail - } - - /** - * Determines whether a JSON contains a specific - * JSON path - * - * @param head The first field in the JSON path - * Exists to ensure the path is nonempty - * @param tail The rest of the fields in the - * JSON path - * @return Whether the path is valid - */ - def fieldExists(config: JValue, head: String, tail: String*): Boolean = - (head +: tail).foldLeft(config)(_ \ _) match { - case JNothing => false - case s => true - } -} diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/Shredder.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/Shredder.scala index 4f1e04f88..d5b2c8e89 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/Shredder.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/Shredder.scala @@ -27,17 +27,13 @@ import Scalaz._ import outputs.EnrichedEvent /** - * The shredder takes the two fields containing JSONs - * (contexts and unstructured event properties) and - * "shreds" their contents into a List of JsonNodes - * ready for loading into dedicated tables in the - * database. + * The shredder takes the two fields containing JSONs (contexts and unstructured event properties) + * and "shreds" their contents into a List of JsonNodes ready for loading into dedicated tables in + * the database. */ object Shredder { - /** - * A (possibly empty) list of JsonNodes - */ + /** A (possibly empty) list of JsonNodes */ type JsonNodes = List[JsonNode] // All shredded JSONs have the events type (aka table) as their ultimate parent @@ -48,31 +44,24 @@ object Shredder { SchemaCriterion("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", 1, 0) // Self-describing schema for a contexts - private val ContextsSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "contexts", "jsonschema", 1, 0) + private val ContextsSchema = + SchemaCriterion("com.snowplowanalytics.snowplow", "contexts", "jsonschema", 1, 0) /** - * Shred the EnrichedEvent's two fields which - * contain JSONs: contexts and unstructured event + * Shred the EnrichedEvent's two fields which contain JSONs: contexts and unstructured event * properties. By shredding we mean: - * * 1. Verify the two fields contain valid JSONs * 2. Validate they conform to JSON Schema - * 3. For the contexts, break the singular JsonNode - * into a List of individual context JsonNodes - * 4. Collect the unstructured event and contexts - * into a singular List - * - * @param event The Snowplow enriched event to - * shred JSONs from - * @param resolver Our implicit Iglu - * Resolver, for schema lookups - * @return a Validation containing on Success a - * List (possible empty) of JsonNodes - * and on Failure a NonEmptyList of - * JsonNodes containing error messages + * 3. For the contexts, break the singular JsonNode into a List of individual context JsonNodes + * 4. Collect the unstructured event and contexts into a singular List + * @param event The Snowplow enriched event to shred JSONs from + * @param resolver Our implicit Iglu Resolver, for schema lookups + * @return a Validation containing on Success a List (possible empty) of JsonNodes and on Failure + * a NonEmptyList of JsonNodes containing error messages */ - def shred(event: EnrichedEvent)(implicit resolver: Resolver): ValidatedNelMessage[JsonSchemaPairs] = { - + def shred(event: EnrichedEvent)( + implicit resolver: Resolver + ): ValidatedNelMessage[List[JsonSchemaPair]] = { // Define what we know so far of the type hierarchy. val partialHierarchy = makePartialHierarchy(event.event_id, event.collector_tstamp) @@ -84,73 +73,72 @@ object Shredder { // Joining all validated JSONs into a single validated List[JsonNode], collecting Failures too val all = ue |+| c |+| dc - val allWithMetadata = all.map { jsonSchemaPairs => + all.map { jsonSchemaPairs => jsonSchemaPairs.map(pair => attachMetadata(pair, partialHierarchy)) } - - allWithMetadata } /** * Extract unstruct event out of EnrichedEvent and validate against it's schema - * * @param event The Snowplow enriched event to find unstruct event in * @param resolver iglu resolver - * @return validated list (empty or single-element) of pairs consist of unstruct event schema and node + * @return validated list (empty or single-element) of pairs consist of unstruct event schema and + * node */ def extractAndValidateUnstructEvent(event: EnrichedEvent)( - implicit resolver: Resolver): ValidatedNelMessage[JsonSchemaPairs] = { + implicit resolver: Resolver + ): ValidatedNelMessage[List[JsonSchemaPair]] = { val extracted: ValidatedNelMessage[JsonNodes] = flatten(extractUnstructEvent(event)) validate(extracted) } /** * Extract list of custom contexts out of string and validate each against its schema - * * @param event The Snowplow enriched event to extract custom context JSONs from * @param resolver iglu resolver * @return validated list of pairs consist of schema and node */ def extractAndValidateCustomContexts(event: EnrichedEvent)( - implicit resolver: Resolver): ValidatedNelMessage[JsonSchemaPairs] = + implicit resolver: Resolver + ): ValidatedNelMessage[List[JsonSchemaPair]] = extractAndValidateContexts(event.contexts, "context") /** * Extract list of derived contexts out of string and validate each against its schema - * * @param event The Snowplow enriched event to extract custom context JSONs from * @param resolver iglu resolver * @return validated list of pairs consist of schema and node */ def extractAndValidateDerivedContexts(event: EnrichedEvent)( - implicit resolver: Resolver): ValidatedNelMessage[JsonSchemaPairs] = + implicit resolver: Resolver + ): ValidatedNelMessage[List[JsonSchemaPair]] = extractAndValidateContexts(event.derived_contexts, "derived_contexts") /** * Extract list of contexts out of string and validate each against its schema - * * @param json string supposed to contain Snowplow Contexts object * @param field field where object is came from (used only for error log) * @param resolver iglu resolver * @return validated list of pairs consist of schema and node */ private[shredder] def extractAndValidateContexts(json: String, field: String)( - implicit resolver: Resolver): ValidatedNelMessage[JsonSchemaPairs] = { + implicit resolver: Resolver + ): ValidatedNelMessage[List[JsonSchemaPair]] = { val extracted: ValidatedNelMessage[JsonNodes] = flatten(extractContexts(json, field)) validate(extracted) } /** - * Extract unstruct event as JsonNode - * Extraction involves validation against schema + * Extract unstruct event as JsonNode. Extraction involves validation against schema. * Event itself extracted as List (empty or with single element) - * * @param event The Snowplow enriched event to shred JSONs from * @param resolver Our implicit Iglu resolver, for schema lookups - * @return a Validation containing on Success a List (possible empty) of JsonNodes - * and on Failure a NonEmptyList of JsonNodes containing error messages + * @return a Validation containing on Success a List (possible empty) of JsonNodes and on Failure + * a NonEmptyList of JsonNodes containing error messages */ - def extractUnstructEvent(event: EnrichedEvent)(implicit resolver: Resolver): Option[ValidatedNelMessage[JsonNodes]] = + def extractUnstructEvent(event: EnrichedEvent)( + implicit resolver: Resolver + ): Option[ValidatedNelMessage[JsonNodes]] = for { v <- extractAndValidateJson("ue_properties", UePropertiesSchema, Option(event.unstruct_event)) } yield @@ -160,17 +148,16 @@ object Shredder { } yield l /** - * Extract list of contexts out of string - * Extraction involves validation against schema - * + * Extract list of contexts out of string. Extraction involves validation against schema * @param json string with contexts object * @param field field where object is came from (used only for error log) * @param resolver Our implicit Iglu resolver, for schema lookups * @return an Optional Validation containing on Success a List (possible empty) of JsonNodes - * and on Failure a NonEmptyList of JsonNodes containing error messages + * and on Failure a NonEmptyList of JsonNodes containing error messages */ private[shredder] def extractContexts(json: String, field: String)( - implicit resolver: Resolver): Option[ValidatedNelMessage[JsonNodes]] = + implicit resolver: Resolver + ): Option[ValidatedNelMessage[JsonNodes]] = for { v <- extractAndValidateJson(field, ContextsSchema, Option(json)) } yield @@ -180,13 +167,14 @@ object Shredder { } yield l /** - * Fetch Iglu Schema for each [[JsonNode]] in [[ValidatedNelMessage]] and validate this node against it - * + * Fetch Iglu Schema for each [[JsonNode]] in [[ValidatedNelMessage]] and validate this node + * against it * @param validatedJsons list of valid JSONs supposed to be Self-describing * @return validated list of pairs consist of schema and node */ private[shredder] def validate(validatedJsons: ValidatedNelMessage[JsonNodes])( - implicit resolver: Resolver): ValidatedNelMessage[JsonSchemaPairs] = { + implicit resolver: Resolver + ): ValidatedNelMessage[List[JsonSchemaPair]] = { val validated = validatedJsons.map { (jsonNodes: List[JsonNode]) => jsonNodes.map(_.validateAndIdentifySchema(false)) } @@ -195,23 +183,21 @@ object Shredder { /** * Flatten Option[List] to List - * * @param o Option with List * @return empty list in case of None, or non-empty in case of some */ - private[shredder] def flatten(o: Option[ValidatedNelMessage[JsonNodes]]): ValidatedNelMessage[JsonNodes] = o match { + private[shredder] def flatten( + o: Option[ValidatedNelMessage[JsonNodes]] + ): ValidatedNelMessage[JsonNodes] = o match { case Some(vjl) => vjl case None => List[JsonNode]().success } /** - * Convenience to make a partial TypeHierarchy. - * Partial because we don't have the complete + * Convenience to make a partial TypeHierarchy. Partial because we don't have the complete * refTree yet. - * * @param rootId The ID of the root element - * @param rootTstamp The timestamp of the root - * element + * @param rootTstamp The timestamp of the root element * @return the partially complete TypeHierarchy */ private[shredder] def makePartialHierarchy(rootId: String, rootTstamp: String): TypeHierarchy = @@ -224,30 +210,22 @@ object Shredder { ) /** - * Adds shred-related metadata to the JSON. - * There are two envelopes of metadata to - * attach: - * - * 1. schema - we replace the existing schema - * URI string with a full schema key object - * containing name, vendor, format and - * version as separate string properties - * 2. hierarchy - we add a new object expressing - * the type hierarchy for this shredded JSON - * + * Adds shred-related metadata to the JSON. There are two envelopes of metadata to attach: + * 1. schema - we replace the existing schema URI string with a full schema key object containing + * name, vendor, format and version as separate string properties + * 2. hierarchy - we add a new object expressing the type hierarchy for this shredded JSON * @param instanceSchemaPair Tuple2 containing: - * 1. The SchemaKey identifying the schema - * for this JSON + * 1. The SchemaKey identifying the schema for this JSON * 2. The JsonNode for this JSON - * @param partialHierarchy The type hierarchy to - * attach. Partial because the refTree is - * still incomplete - * @return the Tuple2, with the JSON updated to - * contain the full schema key, plus the - * now-finalized hierarchy + * @param partialHierarchy The type hierarchy to attach. Partial because the refTree is still + * incomplete + * @return the Tuple2, with the JSON updated to contain the full schema key, plus the + * now-finalized hierarchy */ - private def attachMetadata(instanceSchemaPair: JsonSchemaPair, partialHierarchy: TypeHierarchy): JsonSchemaPair = { - + private def attachMetadata( + instanceSchemaPair: JsonSchemaPair, + partialHierarchy: TypeHierarchy + ): JsonSchemaPair = { val (schemaKey, instance) = instanceSchemaPair val schemaNode = schemaKey.toJsonNode @@ -268,27 +246,19 @@ object Shredder { } /** - * Extract the JSON from a String, and - * validate it against the supplied - * JSON Schema. - * - * @param field The name of the field - * containing the JSON instance - * @param schemaCriterion The criterion we - * expected this self-describing - * JSON to conform to - * @param instance An Option-boxed JSON - * instance - * @param resolver Our implicit Iglu - * Resolver, for schema lookups - * @return an Option-boxed Validation - * containing either a Nel of - * JsonNodes error message on - * Failure, or a singular - * JsonNode on success + * Extract the JSON from a String, and validate it against the supplied JSON Schema. + * @param field The name of the field containing the JSON instance + * @param schemaCriterion The criterion we expected this self-describing JSON to conform to + * @param instance An Option-boxed JSON instance + * @param resolver Our implicit Iglu Resolver, for schema lookups + * @return an Option-boxed Validation containing either a Nel of JsonNodes error message on + * Failure, or a singular JsonNode on success */ - private def extractAndValidateJson(field: String, schemaCriterion: SchemaCriterion, instance: Option[String])( - implicit resolver: Resolver): Option[ValidatedNelMessage[JsonNode]] = + private def extractAndValidateJson( + field: String, + schemaCriterion: SchemaCriterion, + instance: Option[String] + )(implicit resolver: Resolver): Option[ValidatedNelMessage[JsonNode]] = for { i <- instance } yield @@ -298,16 +268,12 @@ object Shredder { } yield v /** - * Wrapper around JsonUtils' extractJson which - * converts the failure to a JsonNode Nel, for - * compatibility with subsequent JSON Schema - * checks. - * - * @param field The name of the field - * containing JSON + * Wrapper around JsonUtils' extractJson which converts the failure to a JsonNode Nel, for + * compatibility with subsequent JSON Schema checks. + * @param field The name of the field containing JSON * @param instance The JSON instance itself * @return the pimped ScalazArgs */ private def extractJson(field: String, instance: String): ValidatedNelMessage[JsonNode] = - JsonUtils.extractJson(field, instance).toProcessingMessageNel + JsonUtils.extractJsonNode(field, instance).toProcessingMessageNel } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/TypeHierarchy.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/TypeHierarchy.scala index f57c14b9b..ac16398b8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/TypeHierarchy.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/shredder/TypeHierarchy.scala @@ -14,23 +14,18 @@ package com.snowplowanalytics.snowplow.enrich.common.utils.shredder import com.fasterxml.jackson.databind.JsonNode import com.github.fge.jackson.JacksonUtils +import io.circe.generic.auto._ +import io.circe.jackson._ +import io.circe.syntax._ import scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ -/** - * Companion object contains helpers. - */ +/** Companion object contains helpers. */ object TypeHierarchy { - private val NodeFactory = JacksonUtils.nodeFactory() } -/** - * Expresses the hierarchy of types for this type. - */ -case class TypeHierarchy( +/** Expresses the hierarchy of types for this type. */ +final case class TypeHierarchy( val rootId: String, val rootTstamp: String, val refRoot: String, @@ -39,44 +34,22 @@ case class TypeHierarchy( ) { /** - * Converts a TypeHierarchy into a JSON containing - * each element. - * + * Converts a TypeHierarchy into a JSON containing each element. * @return the TypeHierarchy as a Jackson JsonNode */ def toJsonNode: JsonNode = - asJsonNode(this.toJValue) - - /** - * Converts a TypeHierarchy into a JSON containing - * each element. - * - * @return the TypeHierarchy as a json4s JValue - */ - def toJValue: JValue = - ("rootId" -> rootId) ~ - ("rootTstamp" -> rootTstamp) ~ - ("refRoot" -> refRoot) ~ - ("refTree" -> refTree) ~ - ("refParent" -> refParent) + circeToJackson(this.asJson) /** - * Completes a partial TypeHierarchy with - * the supplied refTree elements, and uses - * the final refTree to replace the refParent - * too. - * - * @param refTree the rest of the type tree - * to append onto existing refTree + * Completes a partial TypeHierarchy with the supplied refTree elements, and uses + * the final refTree to replace the refParent too. + * @param refTree the rest of the type tree to append onto existing refTree * @return the completed TypeHierarchy */ def complete(refTree: List[String]): TypeHierarchy = partialHierarchyLens.set(this, refTree) - /** - * A Scalaz Lens to complete the refTree within - * a TypeHierarchy object. - */ + /** A Scalaz Lens to complete the refTree within a TypeHierarchy object. */ private val partialHierarchyLens: Lens[TypeHierarchy, List[String]] = Lens.lensu((ph, rt) => { val full = ph.refTree ++ rt @@ -87,11 +60,8 @@ case class TypeHierarchy( }, _.refTree) /** - * Get the last-but-one element ("tail-tail") - * from a list. - * - * @param ls The list to return the last-but-one - * element from + * Get the last-but-one element ("tail-tail") from a list. + * @param ls The list to return the last-but-one element from * @return the last-but-one element from this list */ private def secondTail[A](ls: List[A]): A = ls match { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala index 352becdec..9be35b7c3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala @@ -12,10 +12,7 @@ */ package com.snowplowanalytics.snowplow.enrich -import java.lang.{Integer => JInteger} - import com.github.fge.jsonschema.core.report.ProcessingMessage -import com.snowplowanalytics.iglu.client.JsonSchemaPair import org.apache.http.NameValuePair import scalaz._ @@ -24,122 +21,61 @@ import common.adapters.RawEvent import common.outputs.EnrichedEvent import common.enrichments.registry.Enrichment -/** - * Scala package object to hold types, - * helper methods etc. - * - * See: - * http://www.artima.com/scalazine/articles/package_objects.html - */ +/** Scala package object to hold types, helper methods etc. */ package object common { - /** - * Capture a client's - * screen resolution - */ - type ViewDimensionsTuple = (JInteger, JInteger) // Height, width. - - /** - * Type alias for HTTP headers - */ + /** Type alias for HTTP headers */ type HttpHeaders = List[String] - /** - * Type alias for a map whose - * keys are enrichment names and - * whose values are enrichments - */ + /** Type alias for a map whose keys are enrichment names and whose values are enrichments */ type EnrichmentMap = Map[String, Enrichment] /** - * Type alias for a `ValidationNel` - * containing Strings for `Failure` - * or any type of `Success`. - * + * Type alias for a `ValidationNel` containing Strings for `Failure` or any type of `Success`. * @tparam A the type of `Success` */ type Validated[A] = ValidationNel[String, A] - /** - * Type alias for a `Validation` - * containing either an error - * `String` or a success `String`. - */ + /** Type alias for a `Validation` containing either an error `String` or a success `String`. */ type ValidatedString = Validation[String, String] - /** - * Type alias for a `Validation` - * containing either error `String`s - * or a `NameValueNel`. - */ + /** Type alias for a `Validation` containing either error `String`s or a `NameValueNel`. */ type ValidatedNameValuePairs = Validation[String, List[NameValuePair]] // Note not Validated[] /** - * Type alias for an `Option`-boxed String + * Type alias for either a `ValidationNel` containing `String`s for `Failure` or a + * `MaybeCanonicalInput` for `Success`. */ - type MaybeString = Option[String] + type ValidatedMaybeCollectorPayload = Validated[Option[CollectorPayload]] /** - * Type alias for an `Option`-boxed - * `CollectorPayload`. - */ - type MaybeCollectorPayload = Option[CollectorPayload] - - /** - * Type alias for either a `ValidationNel` - * containing `String`s for `Failure` - * or a `MaybeCanonicalInput` for `Success`. - */ - type ValidatedMaybeCollectorPayload = Validated[MaybeCollectorPayload] - - /** - * Type alias for either a `ValidationNel` - * containing `String`s for `Failure` - * or a `List` of `RawEvent`s for `Success`. + * Type alias for either a `ValidationNel` containing `String`s for `Failure` or a `List` of + * `RawEvent`s for `Success`. */ type ValidatedRawEvents = Validated[NonEmptyList[RawEvent]] /** - * Type alias for an `Option`-boxed - * `CanonicalOutput`. - */ - type MaybeEnrichedEvent = Option[EnrichedEvent] - - /** - * Type alias for either a `ValidationNel` - * containing `String`s for `Failure` - * or a CanonicalOutput for `Success`. + * Type alias for either a `ValidationNel` containing `String`s for `Failure` or a CanonicalOutput + * for `Success`. */ type ValidatedEnrichedEvent = Validated[EnrichedEvent] /** - * Type alias for a `Validation` - * containing ProcessingMessages - * for `Failure` or any type for + * Type alias for a `Validation` containing ProcessingMessages for `Failure` or any type for * `Success` - * * @tparam A the type of `Success` */ type ValidatedMessage[A] = Validation[ProcessingMessage, A] /** - * Type alias for a `ValidationNel` - * containing ProcessingMessage - * for `Failure` or any type for + * Type alias for a `ValidationNel` containing ProcessingMessage for `Failure` or any type for * `Success` */ type ValidatedNelMessage[A] = ValidationNel[ProcessingMessage, A] - /** - * Parameters inside of a raw event - */ + /** Parameters inside of a raw event */ type RawEventParameters = Map[String, String] - /** - * A (possibly empty) list of JsonSchemaPairs - */ - type JsonSchemaPairs = List[JsonSchemaPair] - /** * Type alias for either Throwable or successful value * It has Monad instance unlike Validation diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala index 4dd266647..aac734c6d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala @@ -49,9 +49,10 @@ object SpecHelpers { * the above Iglu configuration. */ val IgluResolver = (for { - json <- JsonUtils.extractJson(igluConfigField, igluConfig) + json <- JsonUtils.extractJsonNode(igluConfigField, igluConfig) reso <- Resolver.parse(json) - } yield reso).getOrElse(throw new RuntimeException("Could not build an Iglu resolver, should never happen")) + } yield reso) + .getOrElse(throw new RuntimeException("Could not build an Iglu resolver, should never happen")) private type NvPair = Tuple2[String, String] diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala index e96a44023..d3d1d2679 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala @@ -15,15 +15,14 @@ package adapters package registry import com.snowplowanalytics.iglu.client.Resolver +import io.circe._ +import io.circe.literal._ import org.joda.time.DateTime import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.JsonMethods._ import SpecHelpers._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} @@ -52,14 +51,15 @@ class AdapterSpec extends Specification with DataTables with ValidationMatchers } object Shared { - val api = CollectorApi("com.adapter", "v1") + val api = CollectorApi("com.adapter", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) val contentType = "application/x-www-form-urlencoded" } @@ -73,30 +73,41 @@ class AdapterSpec extends Specification with DataTables with ValidationMatchers } def e2 = { - val params = BaseAdapter.toUnstructEventParams("tv", Map[String, String](), "iglu:foo", _ => List[JField](), "app") + val params = BaseAdapter.toUnstructEventParams( + "tv", + Map[String, String](), + "iglu:foo", + _ => Json.fromJsonObject(JsonObject.empty), + "app") params must_== Map( - "tv" -> "tv", - "e" -> "ue", - "p" -> "app", + "tv" -> "tv", + "e" -> "ue", + "p" -> "app", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:foo","data":{}}}""" ) } def e3 = { - val shared = Map("nuid" -> "123", - "aid" -> "42", - "cv" -> "clj-tomcat", - "p" -> "srv", - "eid" -> "321", - "ttm" -> "2015-11-13T16:31:52.393Z", - "url" -> "http://localhost") - val params = BaseAdapter.toUnstructEventParams("tv", shared, "iglu:foo", _ => List[JField](), "app") + val shared = Map( + "nuid" -> "123", + "aid" -> "42", + "cv" -> "clj-tomcat", + "p" -> "srv", + "eid" -> "321", + "ttm" -> "2015-11-13T16:31:52.393Z", + "url" -> "http://localhost") + val params = BaseAdapter.toUnstructEventParams( + "tv", + shared, + "iglu:foo", + _ => Json.fromJsonObject(JsonObject.empty), + "app") params must_== shared ++ Map( - "tv" -> "tv", - "e" -> "ue", - "eid" -> "321", - "ttm" -> "2015-11-13T16:31:52.393Z", - "url" -> "http://localhost", + "tv" -> "tv", + "e" -> "ue", + "eid" -> "321", + "ttm" -> "2015-11-13T16:31:52.393Z", + "url" -> "http://localhost", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:foo","data":{}}}""" ) } @@ -107,86 +118,92 @@ class AdapterSpec extends Specification with DataTables with ValidationMatchers } def e5 = - "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED OUTPUT" | - "Failing, nothing passed" !! None ! "Adapter event failed: type parameter not provided - cannot determine event type" | - "Failing, empty type" !! Some("") ! "Adapter event failed: type parameter is empty - cannot determine event type" | - "Failing, bad type passed" !! Some("bad") ! "Adapter event failed: type parameter [bad] not recognized" |> { + "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED OUTPUT" | + "Failing, nothing passed" !! None ! "Adapter event failed: type parameter not provided - cannot determine event type" | + "Failing, empty type" !! Some("") ! "Adapter event failed: type parameter is empty - cannot determine event type" | + "Failing, bad type passed" !! Some("bad") ! "Adapter event failed: type parameter [bad] not recognized" |> { (_, et, expected) => BaseAdapter.lookupSchema(et, "Adapter", SchemaMap) must beFailing(NonEmptyList(expected)) } def e6 = { - val expected = "Adapter event at index [2] failed: type parameter not provided - cannot determine event type" + val expected = + "Adapter event at index [2] failed: type parameter not provided - cannot determine event type" BaseAdapter.lookupSchema(None, "Adapter", 2, SchemaMap) must beFailing(NonEmptyList(expected)) } def e7 = { - val rawEvent = RawEvent(Shared.api, - Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), - Shared.contentType.some, - Shared.cljSource, - Shared.context) + val rawEvent = RawEvent( + Shared.api, + Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), + Shared.contentType.some, + Shared.cljSource, + Shared.context) val validatedRawEventsList = - List(Success(rawEvent), - Failure(NonEmptyList("This is a failure string-1")), - Failure(NonEmptyList("This is a failure string-2"))) + List( + Success(rawEvent), + Failure(NonEmptyList("This is a failure string-1")), + Failure(NonEmptyList("This is a failure string-2"))) val expected = NonEmptyList("This is a failure string-1", "This is a failure string-2") BaseAdapter.rawEventsListProcessor(validatedRawEventsList) must beFailing(expected) } def e8 = { - val rawEvent = RawEvent(Shared.api, - Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), - Shared.contentType.some, - Shared.cljSource, - Shared.context) + val rawEvent = RawEvent( + Shared.api, + Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), + Shared.contentType.some, + Shared.cljSource, + Shared.context) val validatedRawEventsList = List(Success(rawEvent), Success(rawEvent), Success(rawEvent)) val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), - Shared.contentType.some, - Shared.cljSource, - Shared.context), - RawEvent(Shared.api, - Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), - Shared.contentType.some, - Shared.cljSource, - Shared.context), - RawEvent(Shared.api, - Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), - Shared.contentType.some, - Shared.cljSource, - Shared.context) + RawEvent( + Shared.api, + Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), + Shared.contentType.some, + Shared.cljSource, + Shared.context), + RawEvent( + Shared.api, + Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), + Shared.contentType.some, + Shared.cljSource, + Shared.context), + RawEvent( + Shared.api, + Map("tv" -> "com.adapter-v1", "e" -> "ue", "p" -> "srv"), + Shared.contentType.some, + Shared.cljSource, + Shared.context) ) BaseAdapter.rawEventsListProcessor(validatedRawEventsList) must beSuccessful(expected) } def e9 = - "SPEC NAME" || "JSON" | "EXPECTED OUTPUT" | - "Change one value" !! """{"ts":1415709559}""" ! JObject(List(("ts", JString("2014-11-11T12:39:19.000Z")))) | - "Change multiple values" !! """{"ts":1415709559,"ts":1415700000}""" ! JObject( - List(("ts", JString("2014-11-11T12:39:19.000Z")), ("ts", JString("2014-11-11T10:00:00.000Z")))) | - "Change nested values" !! """{"ts":1415709559,"nested":{"ts":1415700000}}""" ! JObject( - List(("ts", JString("2014-11-11T12:39:19.000Z")), - ("nested", JObject(List(("ts", JString("2014-11-11T10:00:00.000Z"))))))) | - "Change nested string values" !! """{"ts":1415709559,"nested":{"ts":"1415700000"}}""" ! JObject( - List(("ts", JString("2014-11-11T12:39:19.000Z")), - ("nested", JObject(List(("ts", JString("2014-11-11T10:00:00.000Z"))))))) | - "JStrings should also be changed" !! """{"ts":"1415709559"}""" ! JObject( - List(("ts", JString("2014-11-11T12:39:19.000Z")))) |> { (_, json, expected) => - BaseAdapter.cleanupJsonEventValues(parse(json), None, "ts") mustEqual expected + "SPEC NAME" || "JSON" | "EXPECTED OUTPUT" | + "Change one value" !! json"""{"ts":1415709559}""" ! json"""{ "ts": "2014-11-11T12:39:19.000Z" }""" | + "Change multiple values" !! json"""{"ts":1415709559,"ts":1415700000}""" ! + json"""{ "ts": "2014-11-11T12:39:19.000Z", "ts": "2014-11-11T10:00:00.000Z"}""" | + "Change nested values" !! json"""{"ts":1415709559,"nested":{"ts":1415700000}}""" ! + json"""{ "ts": "2014-11-11T12:39:19.000Z", "nested": {"ts": "2014-11-11T10:00:00.000Z" }}""" | + "Change nested string values" !! json"""{"ts":1415709559,"nested":{"ts":"1415700000"}}""" ! + json"""{ "ts": "2014-11-11T12:39:19.000Z", "nested": { "ts": "2014-11-11T10:00:00.000Z" }}""" | + "JStrings should also be changed" !! json"""{"ts":"1415709559"}""" ! + json"""{ "ts" : "2014-11-11T12:39:19.000Z" }""" |> { (_, json, expected) => + BaseAdapter.cleanupJsonEventValues(json, None, List("ts")) mustEqual expected } def e10 = - "SPEC NAME" || "JSON" | "EXPECTED OUTPUT" | - "Remove 'event'->'type'" !! """{"an_event":"type"}""" ! JObject(List()) | - "Not remove existing values" !! """{"abc":1415709559, "an_event":"type", "cba":"type"}""" ! JObject( - List(("abc", JInt(1415709559)), ("cba", JString("type")))) | - "Works with ts value subs" !! """{"ts":1415709559, "an_event":"type", "abc":"type"}""" ! JObject( - List(("ts", JString("2014-11-11T12:39:19.000Z")), ("abc", JString("type")))) | - "Removes nested values" !! """{"abc":"abc","nested":{"an_event":"type"}}""" ! JObject( - List(("abc", JString("abc")), ("nested", JObject(List())))) |> { (_, json, expected) => - BaseAdapter.cleanupJsonEventValues(parse(json), ("an_event", "type").some, "ts") mustEqual expected + "SPEC NAME" || "JSON" | "EXPECTED OUTPUT" | + "Remove 'event'->'type'" !! json"""{"an_event":"type"}""" ! json"""{}""" | + "Not remove existing values" !! json"""{"abc":1415709559, "an_event":"type", "cba":"type"}""" ! + json"""{ "abc": 1415709559, "cba": "type" }""" | + "Works with ts value subs" !! json"""{"ts":1415709559, "an_event":"type", "abc":"type"}""" ! + json"""{ "ts": "2014-11-11T12:39:19.000Z", "abc": "type" }""" | + "Removes nested values" !! json"""{"abc":"abc","nested":{"an_event":"type"}}""" ! + json"""{ "abc": "abc", "nested": {}}""" |> { (_, json, expected) => + BaseAdapter.cleanupJsonEventValues(json, ("an_event", "type").some, List("ts")) mustEqual + expected } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala index f9de8fce6..1e24925ba 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala @@ -24,7 +24,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} import SpecHelpers._ -class CallrailAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class CallrailAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the CallrailAdapter functionality toRawEvents should return a NEL containing one RawEvent if the querystring is correctly populated $e1 @@ -34,20 +38,21 @@ class CallrailAdapterSpec extends Specification with DataTables with ValidationM implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.callrail", "v1") + val api = CollectorApi("com.callrail", "v1") val source = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } object Expected { val staticNoPlatform = Map( "tv" -> "com.callrail-v1", - "e" -> "ue", + "e" -> "ue", "cv" -> "clj-0.6.0-tom-0.0.4" ) val static = staticNoPlatform + ("p" -> "srv") @@ -55,46 +60,46 @@ class CallrailAdapterSpec extends Specification with DataTables with ValidationM def e1 = { val params = toNameValuePairs( - "answered" -> "true", - "callercity" -> "BAKERSFIELD", - "callercountry" -> "US", - "callername" -> "SKYPE CALLER", - "callernum" -> "+12612230240", - "callerstate" -> "CA", - "callerzip" -> "92307", - "callsource" -> "keyword", - "datetime" -> "2014-10-09 16:23:45", + "answered" -> "true", + "callercity" -> "BAKERSFIELD", + "callercountry" -> "US", + "callername" -> "SKYPE CALLER", + "callernum" -> "+12612230240", + "callerstate" -> "CA", + "callerzip" -> "92307", + "callsource" -> "keyword", + "datetime" -> "2014-10-09 16:23:45", "destinationnum" -> "2012032051", - "duration" -> "247", - "first_call" -> "true", - "ga" -> "", - "gclid" -> "", - "id" -> "201235151", - "ip" -> "86.178.163.7", - "keywords" -> "", + "duration" -> "247", + "first_call" -> "true", + "ga" -> "", + "gclid" -> "", + "id" -> "201235151", + "ip" -> "86.178.163.7", + "keywords" -> "", "kissmetrics_id" -> "", - "landingpage" -> "http://acme.com/", - "recording" -> "http://app.callrail.com/calls/201235151/recording/9f59ad59ba1cfa264312", - "referrer" -> "direct", + "landingpage" -> "http://acme.com/", + "recording" -> "http://app.callrail.com/calls/201235151/recording/9f59ad59ba1cfa264312", + "referrer" -> "direct", "referrermedium" -> "Direct", - "trackingnum" -> "+12012311668", - "transcription" -> "", - "utm_campaign" -> "", - "utm_content" -> "", - "utm_medium" -> "", - "utm_source" -> "", - "utm_term" -> "", - "utma" -> "", - "utmb" -> "", - "utmc" -> "", - "utmv" -> "", - "utmx" -> "", - "utmz" -> "", - "cv" -> "clj-0.6.0-tom-0.0.4", - "nuid" -> "-" + "trackingnum" -> "+12012311668", + "transcription" -> "", + "utm_campaign" -> "", + "utm_content" -> "", + "utm_medium" -> "", + "utm_source" -> "", + "utm_term" -> "", + "utma" -> "", + "utmb" -> "", + "utmc" -> "", + "utmv" -> "", + "utmx" -> "", + "utmz" -> "", + "cv" -> "clj-0.6.0-tom-0.0.4", + "nuid" -> "-" ) val payload = CollectorPayload(Shared.api, params, None, None, Shared.source, Shared.context) - val actual = CallrailAdapter.toRawEvents(payload) + val actual = CallrailAdapter.toRawEvents(payload) val expectedJson = """|{ @@ -143,17 +148,18 @@ class CallrailAdapterSpec extends Specification with DataTables with ValidationM actual must beSuccessful( NonEmptyList( - RawEvent(Shared.api, - Expected.static ++ Map("ue_pr" -> expectedJson, "nuid" -> "-"), - None, - Shared.source, - Shared.context))) + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson, "nuid" -> "-"), + None, + Shared.source, + Shared.context))) } def e2 = { - val params = toNameValuePairs() + val params = toNameValuePairs() val payload = CollectorPayload(Shared.api, params, None, None, Shared.source, Shared.context) - val actual = CallrailAdapter.toRawEvents(payload) + val actual = CallrailAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList("Querystring is empty: no CallRail event to process")) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala index ed7721b88..9d54a76f8 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala @@ -24,7 +24,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource, TsvLoader} import SpecHelpers._ -class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class CloudfrontAccessLogAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the CloudfrontAccessLogAdapter functionality toRawEvents should return a NEL containing one RawEvent if the line contains 12 fields $e1 @@ -39,7 +43,7 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with """ implicit val resolver = SpecHelpers.IgluResolver - val loader = new TsvLoader("com.amazon.aws.cloudfront/wd_access_log") + val loader = new TsvLoader("com.amazon.aws.cloudfront/wd_access_log") val doubleEncodedUa = "Mozilla/5.0%2520(Macintosh;%2520Intel%2520Mac%2520OS%2520X%252010_9_2)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/34.0.1847.131%2520Safari/537.36" @@ -54,21 +58,22 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with val url = "http://snowplowanalytics.com/analytics/index.html" object Shared { - val api = CollectorApi("com.amazon.aws.cloudfront", "wd_access_log") + val api = CollectorApi("com.amazon.aws.cloudfront", "wd_access_log") val source = CollectorSource("tsv", "UTF-8", None) val context = - CollectorContext(DateTime.parse("2013-10-07T23:35:30.000Z").some, - "255.255.255.255".some, - singleEncodedUa.some, - None, - Nil, - None) + CollectorContext( + DateTime.parse("2013-10-07T23:35:30.000Z").some, + "255.255.255.255".some, + singleEncodedUa.some, + None, + Nil, + None) } object Expected { val staticNoPlatform = Map( - "tv" -> "com.amazon.aws.cloudfront/wd_access_log", - "e" -> "ue", + "tv" -> "com.amazon.aws.cloudfront/wd_access_log", + "e" -> "ue", "url" -> url ) val static = staticNoPlatform ++ Map( @@ -78,7 +83,8 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with def e1 = { - val input = s"2013-10-07\t23:35:30\tc\t100\t255.255.255.255\tf\tg\th\ti\t$url\t$doubleEncodedUa\t$doubleEncodedQs" + val input = + s"2013-10-07\t23:35:30\tc\t100\t255.255.255.255\tf\tg\th\ti\t$url\t$doubleEncodedUa\t$doubleEncodedQs" val payload = loader.toCollectorPayload(input) @@ -106,8 +112,15 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with |}""".stripMargin.replaceAll("[\n\r]", "") actual must beSuccessful( - Some(Success(NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.source, Shared.context))))) + Some( + Success( + NonEmptyList( + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.source, + Shared.context))))) } def e2 = { @@ -144,8 +157,15 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with |}""".stripMargin.replaceAll("[\n\r]", "") actual must beSuccessful( - Some(Success(NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.source, Shared.context))))) + Some( + Success( + NonEmptyList( + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.source, + Shared.context))))) } def e3 = { @@ -185,8 +205,15 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with |}""".stripMargin.replaceAll("[\n\r]", "") actual must beSuccessful( - Some(Success(NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.source, Shared.context))))) + Some( + Success( + NonEmptyList( + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.source, + Shared.context))))) } def e4 = { @@ -227,8 +254,15 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with |}""".stripMargin.replaceAll("[\n\r]", "") actual must beSuccessful( - Some(Success(NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.source, Shared.context))))) + Some( + Success( + NonEmptyList( + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.source, + Shared.context))))) } def e5 = { @@ -273,8 +307,15 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with |}""".stripMargin.replaceAll("[\n\r]", "") actual must beSuccessful( - Some(Success(NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.source, Shared.context))))) + Some( + Success( + NonEmptyList( + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.source, + Shared.context))))) } def e6 = { @@ -320,8 +361,15 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with |}""".stripMargin.replaceAll("[\n\r]", "") actual must beSuccessful( - Some(Success(NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.source, Shared.context))))) + Some( + Success( + NonEmptyList( + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.source, + Shared.context))))) } def e7 = { @@ -369,28 +417,43 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with |}""".stripMargin.replaceAll("[\n\r]", "") actual must beSuccessful( - Some(Success(NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.source, Shared.context))))) + Some( + Success( + NonEmptyList( + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.source, + Shared.context))))) } def e8 = { val params = toNameValuePairs() val payload = - CollectorPayload(Shared.api, params, None, "2013-10-07\t23:35:30\tc\t\t".some, Shared.source, Shared.context) + CollectorPayload( + Shared.api, + params, + None, + "2013-10-07\t23:35:30\tc\t\t".some, + Shared.source, + Shared.context) val actual = CloudfrontAccessLogAdapter.WebDistribution.toRawEvents(payload) - actual must beFailing(NonEmptyList("Access log TSV line contained 5 fields, expected 12, 15, 18, 19, 23, 24 or 26")) + actual must beFailing( + NonEmptyList("Access log TSV line contained 5 fields, expected 12, 15, 18, 19, 23, 24 or 26")) } def e9 = { val params = toNameValuePairs() val payload = - CollectorPayload(Shared.api, - params, - None, - s"a\tb\tc\td\te\tf\tg\th\ti\t$url\tk\t$doubleEncodedQs".some, - Shared.source, - Shared.context) + CollectorPayload( + Shared.api, + params, + None, + s"a\tb\tc\td\te\tf\tg\th\ti\t$url\tk\t$doubleEncodedQs".some, + Shared.source, + Shared.context) val actual = CloudfrontAccessLogAdapter.WebDistribution.toRawEvents(payload) actual must beFailing( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala index b6f41331f..1df58c4a3 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala @@ -53,15 +53,21 @@ class GoogleAnalyticsAdapterSpec implicit val resolver = SpecHelpers.IgluResolver - val api = CollectorApi("com.google.analytics.measurement-protocol", "v1") + val api = CollectorApi("com.google.analytics.measurement-protocol", "v1") val source = CollectorSource("clj-tomcat", "UTF-8", None) val context = - CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, "37.157.33.123".some, None, None, Nil, None) + CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) val static = Map( "tv" -> "com.google.analytics.measurement-protocol-v1", - "e" -> "ue", - "p" -> "srv" + "e" -> "ue", + "p" -> "srv" ) val hitContext = (hitType: String) => s""" @@ -72,30 +78,33 @@ class GoogleAnalyticsAdapterSpec def e1 = { val payload = CollectorPayload(api, Nil, None, None, source, context) - val actual = toRawEvents(payload) - actual must beFailing(NonEmptyList("Request body is empty: no GoogleAnalytics events to process")) + val actual = toRawEvents(payload) + actual must beFailing( + NonEmptyList("Request body is empty: no GoogleAnalytics events to process")) } def e2 = { - val body = "dl=docloc" + val body = "dl=docloc" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) - actual must beFailing(NonEmptyList("No GoogleAnalytics t parameter provided: cannot determine hit type")) + val actual = toRawEvents(payload) + actual must beFailing( + NonEmptyList("No GoogleAnalytics t parameter provided: cannot determine hit type")) } def e3 = { - val body = "t=unknown&dl=docloc" + val body = "t=unknown&dl=docloc" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) actual must beFailing( - NonEmptyList("No matching GoogleAnalytics hit type for hit type unknown", - "GoogleAnalytics event failed: type parameter [unknown] not recognized")) + NonEmptyList( + "No matching GoogleAnalytics hit type for hit type unknown", + "GoogleAnalytics event failed: type parameter [unknown] not recognized")) } def e4 = { - val body = "t=pageview&dh=host&dp=path" + val body = "t=pageview&dh=host&dp=path" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedJson = """|{ @@ -118,9 +127,9 @@ class GoogleAnalyticsAdapterSpec } def e5 = { - val body = "t=pageview&dh=host&cid=id&v=version" + val body = "t=pageview&dh=host&cid=id&v=version" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -148,9 +157,9 @@ class GoogleAnalyticsAdapterSpec } def e6 = { - val body = "t=pageview&dp=path&uip=ip" + val body = "t=pageview&dp=path&uip=ip" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -174,9 +183,9 @@ class GoogleAnalyticsAdapterSpec } def e7 = { - val body = "t=item&in=name&ip=12.228&iq=12&aip=0" + val body = "t=item&in=name&ip=12.228&iq=12&aip=0" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -198,19 +207,20 @@ class GoogleAnalyticsAdapterSpec |"data":{"anonymizeIp":false} |},${hitContext("item")}] |}""".stripMargin.replaceAll("[\n\r]", "") - val expectedParams = static ++ Map("ue_pr" -> expectedUE, - "co" -> expectedCO, - // ip, iq and in are direct mappings too - "ti_pr" -> "12.228", - "ti_qu" -> "12", - "ti_nm" -> "name") + val expectedParams = static ++ Map( + "ue_pr" -> expectedUE, + "co" -> expectedCO, + // ip, iq and in are direct mappings too + "ti_pr" -> "12.228", + "ti_qu" -> "12", + "ti_nm" -> "name") actual must beSuccessful(NonEmptyList(RawEvent(api, expectedParams, None, source, context))) } def e8 = { - val body = "t=exception&exd=desc&exf=1&dh=host" + val body = "t=exception&exd=desc&exf=1&dh=host" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -233,9 +243,9 @@ class GoogleAnalyticsAdapterSpec } def e9 = { - val body = "t=transaction&ti=tr&cu=EUR&pr12id=ident&pr12cd42=val" + val body = "t=transaction&ti=tr&cu=EUR&pr12id=ident&pr12cd42=val" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -256,14 +266,18 @@ class GoogleAnalyticsAdapterSpec |"data":{"currencyCode":"EUR","sku":"ident","index":12} |}] |}""".stripMargin.replaceAll("[\n\r]", "") - val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO, "tr_cu" -> "EUR", "tr_id" -> "tr") + val expectedParams = static ++ Map( + "ue_pr" -> expectedUE, + "co" -> expectedCO, + "tr_cu" -> "EUR", + "tr_id" -> "tr") actual must beSuccessful(NonEmptyList(RawEvent(api, expectedParams, None, source, context))) } def e10 = { - val body = "t=pageview&dp=path&il12pi42id=s&il12pi42cd36=dim" + val body = "t=pageview&dp=path&il12pi42id=s&il12pi42cd36=dim" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -289,9 +303,9 @@ class GoogleAnalyticsAdapterSpec } def e11 = { - val body = "t=screenview&cd=name&cd12=dim" + val body = "t=screenview&cd=name&cd12=dim" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -314,9 +328,9 @@ class GoogleAnalyticsAdapterSpec } def e12 = { - val body = "t=pageview&dp=path&pr1id=s1&pr2id=s2&pr1cd1=v1&pr1cd2=v2" + val body = "t=pageview&dp=path&pr1id=s1&pr2id=s2&pr1cd1=v1&pr1cd2=v2" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -348,9 +362,9 @@ class GoogleAnalyticsAdapterSpec } def e13 = { - val body = "t=pageview&dp=path&promoa=action&promo12id=id" + val body = "t=pageview&dp=path&promoa=action&promo12id=id" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedUE = """|{ @@ -376,9 +390,9 @@ class GoogleAnalyticsAdapterSpec } def e14 = { - val body = "t=pageview&dh=host&dp=path\nt=pageview&dh=host&dp=path" + val body = "t=pageview&dh=host&dp=path\nt=pageview&dh=host&dp=path" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedJson = """|{ @@ -397,7 +411,7 @@ class GoogleAnalyticsAdapterSpec |"data":[${hitContext("pageview")}] |}""".stripMargin.replaceAll("[\n\r]", "") val expectedParams = static ++ Map("ue_pr" -> expectedJson, "co" -> expectedCO) - val event = RawEvent(api, expectedParams, None, source, context) + val event = RawEvent(api, expectedParams, None, source, context) actual must beSuccessful(NonEmptyList(event, event)) } @@ -405,7 +419,7 @@ class GoogleAnalyticsAdapterSpec val body = "t=pageview&dh=host&dp=path&cu=EUR&il1pi1pr=1&il1pi1nm=name1&il1pi1ps=1&il1pi1ca=cat1&il1pi1id=id1&il1pi1br=brand1&il1pi2pr=2&il1pi2nm=name2&il1pi2ps=2&il1pi2ca=cat2&il1pi2id=id2&il1pi2br=brand2&il2pi1pr=21&il2pi1nm=name21&il2pi1ps=21&il2pi1ca=cat21&il2pi1id=id21&il2pi1br=brand21" val payload = CollectorPayload(api, Nil, None, body.some, source, context) - val actual = toRawEvents(payload) + val actual = toRawEvents(payload) val expectedJson = """|{ @@ -432,7 +446,10 @@ class GoogleAnalyticsAdapterSpec |"data":{"productIndex":1,"name":"name21","sku":"id21","price":21.0,"brand":"brand21","currencyCode":"EUR","category":"cat21","position":21,"listIndex":2} |}] |}""".stripMargin.replaceAll("[\n\r]", "") - val expectedParams = static ++ Map("ue_pr" -> expectedJson, "co" -> expectedCO, "ti_cu" -> "EUR") + val expectedParams = static ++ Map( + "ue_pr" -> expectedJson, + "co" -> expectedCO, + "ti_cu" -> "EUR") actual must beSuccessful(NonEmptyList(RawEvent(api, expectedParams, None, source, context))) } @@ -451,7 +468,8 @@ class GoogleAnalyticsAdapterSpec Map("IFprcm" -> "12", "IFcm" -> "42", "prcm" -> "value")), breakDownCompField("pr", "value", "IF") must beLeftDisjunction(errorMessage("pr")), breakDownCompField("pr", "", "IF") must beLeftDisjunction(errorMessage("pr")), - breakDownCompField("pr12", "val", "IF") must beRightDisjunction(Map("IFpr" -> "12", "pr" -> "val")) + breakDownCompField("pr12", "val", "IF") must beRightDisjunction( + Map("IFpr" -> "12", "pr" -> "val")) ) s.reduce(_ and _) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala index d2809d4d3..ff6cfbe29 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala @@ -14,17 +14,21 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry +import io.circe.literal._ import org.joda.time.DateTime import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} -class HubSpotAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class HubSpotAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the HubSpotAdapter functionality payloadBodyToEvents must return a Success list of event JSON's from a valid payload body $e1 @@ -39,27 +43,31 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidationMa implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.hubspot", "v1") + val api = CollectorApi("com.hubspot", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/json" def e1 = { - val bodyStr = """[{"subscriptionType":"company.change","eventId":16}]""" - val expected = List(JObject(List(("subscriptionType", JString("company.change")), ("eventId", JInt(16))))) - HubSpotAdapter.payloadBodyToEvents(bodyStr) must beSuccessful(expected) + val bodyStr = """[{"subscriptionType":"company.change","eventId":16}]""" + val expected = json"""{ + "subscriptionType": "company.change", + "eventId": 16 + }""" + HubSpotAdapter.payloadBodyToEvents(bodyStr) must beSuccessful(List(expected)) } def e2 = - "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | - "Failure, parse exception" !! """{"something:"some"}""" ! """HubSpot payload failed to parse into JSON: [com.fasterxml.jackson.core.JsonParseException: Unexpected character ('s' (code 115)): was expecting a colon to separate field name and value at [Source: (String)"{"something:"some"}"; line: 1, column: 15]]""" |> { + "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | + "Failure, parse exception" !! """{"something:"some"}""" ! """HubSpot payload failed to parse into JSON: [expected : got 'some"}' (line 1, column 14)]""" |> { (_, input, expected) => HubSpotAdapter.payloadBodyToEvents(input) must beFailing(expected) } @@ -67,14 +75,20 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidationMa def e3 = { val bodyStr = """[{"eventId":1,"subscriptionId":25458,"portalId":4737818,"occurredAt":1539145399845,"subscriptionType":"contact.creation","attemptNumber":0,"objectId":123,"changeSource":"CRM","changeFlag":"NEW","appId":177698}]""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.hubspot-v1", - "e" -> "ue", - "p" -> "srv", + "tv" -> "com.hubspot-v1", + "e" -> "ue", + "p" -> "srv", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.hubspot/contact_creation/jsonschema/1-0-0","data":{"eventId":1,"subscriptionId":25458,"portalId":4737818,"occurredAt":"2018-10-10T04:23:19.845Z","attemptNumber":0,"objectId":123,"changeSource":"CRM","changeFlag":"NEW","appId":177698}}}""" ), ContentType.some, @@ -87,31 +101,41 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidationMa def e4 = { val bodyStr = """[{"eventId":1,"subscriptionId":25458,"portalId":4737818,"occurredAt":1539145399845,"subscriptionType":"contact","attemptNumber":0,"objectId":123,"changeSource":"CRM","changeFlag":"NEW","appId":177698}]""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = "HubSpot event at index [0] failed: type parameter [contact] not recognized" HubSpotAdapter.toRawEvents(payload) must beFailing(NonEmptyList(expected)) } def e5 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) HubSpotAdapter.toRawEvents(payload) must beFailing( NonEmptyList("Request body is empty: no HubSpot events to process")) } def e6 = { - val payload = CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) HubSpotAdapter.toRawEvents(payload) must beFailing( - NonEmptyList("Request body provided but content type empty, expected application/json for HubSpot")) + NonEmptyList( + "Request body provided but content type empty, expected application/json for HubSpot")) } def e7 = { - val payload = CollectorPayload(Shared.api, - Nil, - "application/x-www-form-urlencoded".some, - "stub".some, - Shared.cljSource, - Shared.context) - HubSpotAdapter.toRawEvents(payload) must beFailing( - NonEmptyList("Content type of application/x-www-form-urlencoded provided, expected application/json for HubSpot")) + val payload = CollectorPayload( + Shared.api, + Nil, + "application/x-www-form-urlencoded".some, + "stub".some, + Shared.cljSource, + Shared.context) + HubSpotAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Content type of application/x-www-form-urlencoded provided, expected application/json for HubSpot")) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala index 4eac02c35..d840fbcb8 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala @@ -24,7 +24,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} import SpecHelpers._ -class IgluAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class IgluAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the IgluAdapter functionality toRawEvents should return a NEL containing one RawEvent if the CloudFront querystring is minimally populated $e1 @@ -49,21 +53,22 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.snowplowanalytics.iglu", "v1") - val cfSource = CollectorSource("cloudfront", "UTF-8", None) + val api = CollectorApi("com.snowplowanalytics.iglu", "v1") + val cfSource = CollectorSource("cloudfront", "UTF-8", None) val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } object Expected { val staticNoPlatform = Map( "tv" -> "com.snowplowanalytics.iglu-v1", - "e" -> "ue" + "e" -> "ue" ) val static = staticNoPlatform ++ Map( "p" -> "app" @@ -72,16 +77,16 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch def e1 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-0", - "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", - "name" -> "download", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-0", + "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", + "name" -> "download", "publisher_name" -> "Organic", - "source" -> "", - "tracking_id" -> "", - "ad_unit" -> "" + "source" -> "", + "tracking_id" -> "", + "ad_unit" -> "" ) val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val actual = IgluAdapter.toRawEvents(payload) val expectedJson = """|{ @@ -101,21 +106,26 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch actual must beSuccessful( NonEmptyList( - RawEvent(Shared.api, Expected.static ++ Map("ue_pr" -> expectedJson), None, Shared.cfSource, Shared.context))) + RawEvent( + Shared.api, + Expected.static ++ Map("ue_pr" -> expectedJson), + None, + Shared.cfSource, + Shared.context))) } def e2 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-0", - "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", - "name" -> "install", - "source" -> "newsfeed", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-0", + "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", + "name" -> "install", + "source" -> "newsfeed", "tracking_id" -> "3353af9c-e298-2cf3-9f1c-b377a9c84dad", - "ad_unit" -> "UN-11-b", - "aid" -> "webhooks" + "ad_unit" -> "UN-11-b", + "aid" -> "webhooks" ) val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val actual = IgluAdapter.toRawEvents(payload) val expectedMap = { val json = @@ -134,28 +144,28 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch |}""".stripMargin.replaceAll("[\n\r]", "") Map( "ue_pr" -> json, - "aid" -> "webhooks" + "aid" -> "webhooks" ) } - actual must beSuccessful( - NonEmptyList(RawEvent(Shared.api, Expected.static ++ expectedMap, None, Shared.cfSource, Shared.context))) + actual must beSuccessful(NonEmptyList( + RawEvent(Shared.api, Expected.static ++ expectedMap, None, Shared.cfSource, Shared.context))) } def e3 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/2-0-0", - "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", - "name" -> "retarget", - "source" -> "newsfeed", + "schema" -> "iglu:com.acme/campaign/jsonschema/2-0-0", + "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", + "name" -> "retarget", + "source" -> "newsfeed", "tracking_id" -> "", - "ad_unit" -> "UN-11-b", - "aid" -> "my webhook project", - "cv" -> "clj-0.5.0-tom-0.0.4", - "nuid" -> "" + "ad_unit" -> "UN-11-b", + "aid" -> "my webhook project", + "cv" -> "clj-0.5.0-tom-0.0.4", + "nuid" -> "" ) val payload = CollectorPayload(Shared.api, params, None, None, Shared.cljSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val actual = IgluAdapter.toRawEvents(payload) val expectedMap = { val json = @@ -174,25 +184,25 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch |}""".stripMargin.replaceAll("[\n\r]", "") Map( "ue_pr" -> json, - "aid" -> "my webhook project", - "cv" -> "clj-0.5.0-tom-0.0.4", - "nuid" -> "" + "aid" -> "my webhook project", + "cv" -> "clj-0.5.0-tom-0.0.4", + "nuid" -> "" ) } - actual must beSuccessful( - NonEmptyList(RawEvent(Shared.api, Expected.static ++ expectedMap, None, Shared.cljSource, Shared.context))) + actual must beSuccessful(NonEmptyList( + RawEvent(Shared.api, Expected.static ++ expectedMap, None, Shared.cljSource, Shared.context))) } def e4 = { val params = toNameValuePairs( "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", - "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", - "name" -> "download", - "p" -> "mob" + "user" -> "6353af9b-e288-4cf3-9f1c-b377a9c84dac", + "name" -> "download", + "p" -> "mob" ) val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val actual = IgluAdapter.toRawEvents(payload) val expectedJson = """|{ @@ -208,30 +218,33 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch actual must beSuccessful( NonEmptyList( - RawEvent(Shared.api, - Expected.staticNoPlatform ++ Map("p" -> "mob", "ue_pr" -> expectedJson), - None, - Shared.cfSource, - Shared.context))) + RawEvent( + Shared.api, + Expected.staticNoPlatform ++ Map("p" -> "mob", "ue_pr" -> expectedJson), + None, + Shared.cfSource, + Shared.context))) } def e5 = { - val params = toNameValuePairs() + val params = toNameValuePairs() val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val actual = IgluAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Iglu event failed: is not a sd-json or a valid GET or POST request")) + actual must beFailing( + NonEmptyList("Iglu event failed: is not a sd-json or a valid GET or POST request")) } def e6 = { val params = toNameValuePairs( "some_param" -> "foo", - "p" -> "mob" + "p" -> "mob" ) val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val actual = IgluAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Iglu event failed: is not a sd-json or a valid GET or POST request")) + actual must beFailing( + NonEmptyList("Iglu event failed: is not a sd-json or a valid GET or POST request")) } def e7 = { @@ -239,28 +252,34 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch "schema" -> "iglooooooo://blah" ) val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val actual = IgluAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList("iglooooooo://blah is not a valid Iglu-format schema URI")) } def e8 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", "some_param" -> "foo", - "p" -> "mob" + "p" -> "mob" ) val jsonStr = """{"key":"value", "everwets":"processed"}""" val payload = - CollectorPayload(Shared.api, params, "application/json".some, jsonStr.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + params, + "application/json".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) val expected = RawEvent( Shared.api, Map( - "tv" -> "com.snowplowanalytics.iglu-v1", - "e" -> "ue", - "p" -> "mob", + "tv" -> "com.snowplowanalytics.iglu-v1", + "e" -> "ue", + "p" -> "mob", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1","data":{"key":"value","everwets":"processed"}}}""" ), "application/json".some, @@ -273,13 +292,19 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch def e9 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", "some_param" -> "foo", - "p" -> "mob" + "p" -> "mob" ) val jsonStr = """{"key":"value", "everwets":"processed"}""" val payload = - CollectorPayload(Shared.api, params, "application/badtype".some, jsonStr.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + params, + "application/badtype".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList("Content type not supported")) @@ -287,44 +312,60 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch def e10 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", "some_param" -> "foo", - "p" -> "mob" + "p" -> "mob" ) val jsonStr = """{}""" val payload = - CollectorPayload(Shared.api, params, "application/json".some, jsonStr.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + params, + "application/json".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Iglu event failed json sanity check: has no key-value pairs")) + actual must beFailing( + NonEmptyList("Iglu event failed json sanity check: has no key-value pairs")) } def e11 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", "some_param" -> "foo", - "p" -> "mob" + "p" -> "mob" ) val jsonStr = """{"key":"value"}""" - val payload = CollectorPayload(Shared.api, params, None, jsonStr.some, Shared.cljSource, Shared.context) - val actual = IgluAdapter.toRawEvents(payload) + val payload = + CollectorPayload(Shared.api, params, None, jsonStr.some, Shared.cljSource, Shared.context) + val actual = IgluAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Iglu event failed: ContentType must be set for a POST payload")) + actual must beFailing( + NonEmptyList("Iglu event failed: ContentType must be set for a POST payload")) } def e12 = { - val params = toNameValuePairs("p" -> "mob") - val jsonStr = """{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1", "data":{"some_param":"foo"}}""" + val params = toNameValuePairs("p" -> "mob") + val jsonStr = + """{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1", "data":{"some_param":"foo"}}""" val payload = - CollectorPayload(Shared.api, params, "application/json".some, jsonStr.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + params, + "application/json".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) val expected = RawEvent( Shared.api, Map( - "tv" -> "com.snowplowanalytics.iglu-v1", - "e" -> "ue", - "p" -> "mob", + "tv" -> "com.snowplowanalytics.iglu-v1", + "e" -> "ue", + "p" -> "mob", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1","data":{"some_param":"foo"}}}""" ), "application/json".some, @@ -336,53 +377,57 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch } def e13 = { - val params = toNameValuePairs("p" -> "mob") - val jsonStr = """{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1", "data":{"some_param":"foo"}}""" + val params = toNameValuePairs("p" -> "mob") + val jsonStr = + """{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1", "data":{"some_param":"foo"}}""" val payload = - CollectorPayload(Shared.api, - params, - "application/xxx-url-form-encoded".some, - jsonStr.some, - Shared.cljSource, - Shared.context) + CollectorPayload( + Shared.api, + params, + "application/xxx-url-form-encoded".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList("Content type not supported")) } def e14 = { - val params = toNameValuePairs("schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1") + val params = toNameValuePairs("schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1") val jsonStr = """{"some_param":"foo"}""" val payload = - CollectorPayload(Shared.api, - params, - "application/xxx-url-form-encoded".some, - jsonStr.some, - Shared.cljSource, - Shared.context) + CollectorPayload( + Shared.api, + params, + "application/xxx-url-form-encoded".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList("Content type not supported")) } def e15 = { - val params = toNameValuePairs("schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1") + val params = toNameValuePairs("schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1") val formBodyStr = "some_param=foo&hello=world" val payload = - CollectorPayload(Shared.api, - params, - "application/x-www-form-urlencoded".some, - formBodyStr.some, - Shared.cljSource, - Shared.context) + CollectorPayload( + Shared.api, + params, + "application/x-www-form-urlencoded".some, + formBodyStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) val expected = RawEvent( Shared.api, Map( - "tv" -> "com.snowplowanalytics.iglu-v1", - "e" -> "ue", - "p" -> "srv", + "tv" -> "com.snowplowanalytics.iglu-v1", + "e" -> "ue", + "p" -> "srv", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1","data":{"some_param":"foo","hello":"world"}}}""" ), "application/x-www-form-urlencoded".some, @@ -395,21 +440,28 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch def e16 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", "some_param" -> "foo", - "p" -> "mob" + "p" -> "mob" ) - val jsonStr = """[{"key":"value", "everwets":"processed"},{"key":"value", "everwets":"processed"}]""" + val jsonStr = + """[{"key":"value", "everwets":"processed"},{"key":"value", "everwets":"processed"}]""" val payload = - CollectorPayload(Shared.api, params, "application/json".some, jsonStr.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + params, + "application/json".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) val expected = RawEvent( Shared.api, Map( - "tv" -> "com.snowplowanalytics.iglu-v1", - "e" -> "ue", - "p" -> "mob", + "tv" -> "com.snowplowanalytics.iglu-v1", + "e" -> "ue", + "p" -> "mob", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.acme/campaign/jsonschema/1-0-1","data":{"key":"value","everwets":"processed"}}}""" ), "application/json".some, @@ -422,15 +474,22 @@ class IgluAdapterSpec extends Specification with DataTables with ValidationMatch def e17 = { val params = toNameValuePairs( - "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", + "schema" -> "iglu:com.acme/campaign/jsonschema/1-0-1", "some_param" -> "foo", - "p" -> "mob" + "p" -> "mob" ) val jsonStr = """[]""" val payload = - CollectorPayload(Shared.api, params, "application/json".some, jsonStr.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + params, + "application/json".some, + jsonStr.some, + Shared.cljSource, + Shared.context) val actual = IgluAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Iglu event failed json sanity check: array of events cannot be empty")) + actual must beFailing( + NonEmptyList("Iglu event failed json sanity check: array of events cannot be empty")) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala index 067b83dd8..c39adfe6f 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala @@ -14,24 +14,29 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry +import io.circe._ +import io.circe.literal._ import org.joda.time.DateTime -import org.specs2.{Specification, ScalaCheck} +import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.JsonDSL._ import loaders._ import SpecHelpers._ -class MailchimpAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { def is = s2""" +class MailchimpAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { + def is = s2""" This is a specification to test the MailchimpAdapter functionality toKeys should return a valid List of Keys from a string containing braces (or not) $e1 - toNestedJField should return a valid JField nested to contain all keys and then the supplied value $e2 - toJFields should return a valid list of JFields based on the Map supplied $e3 - mergeJFields should return a correctly merged JSON which matches the expectation $e4 + toNestedJson should return a valid JField nested to contain all keys and then the supplied value $e2 + toJsons should return a valid list of Jsons based on the Map supplied $e3 + mergeJsons should return a correctly merged JSON which matches the expectation $e4 reformatParameters should return a parameter Map with correctly formatted values $e5 toRawEvents must return a Nel Success with a correctly formatted ue_pr json $e6 toRawEvents must return a Nel Success with a correctly merged and formatted ue_pr json $e7 @@ -49,7 +54,13 @@ class MailchimpAdapterSpec extends Specification with DataTables with Validation object Shared { val api = CollectorApi("com.mailchimp", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, "37.157.33.123".some, None, None, Nil, None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/x-www-form-urlencoded" @@ -64,9 +75,9 @@ class MailchimpAdapterSpec extends Specification with DataTables with Validation def e2 = { val keys = NonEmptyList("data", "merges", "LNAME") val value = "Beemster" - val expected = JField("data", JObject(List(("merges", JObject(List(("LNAME", JString("Beemster")))))))) + val expected = ("data", json"""{ "merges": { "LNAME": "Beemster" }}""") - MailchimpAdapter.toNestedJField(keys, value) mustEqual expected + MailchimpAdapter.toNestedJson(keys, value) mustEqual expected } def e3 = { @@ -75,111 +86,170 @@ class MailchimpAdapterSpec extends Specification with DataTables with Validation "data[merges][FNAME]" -> "Joshua" ) val expected = List( - JField("data", JObject(List(("merges", JObject(List(("LNAME", JString("Beemster")))))))), - JField("data", JObject(List(("merges", JObject(List(("FNAME", JString("Joshua")))))))) + ("data", json"""{ "merges": { "LNAME": "Beemster" }}"""), + ("data", json"""{ "merges": { "FNAME": "Joshua" }}""") ) - - MailchimpAdapter.toJFields(map) mustEqual expected + MailchimpAdapter.toJsons(map) mustEqual expected } def e4 = { - val a = JField("l1", JField("l2", JField("l3", JField("str", "hi")))) - val b = JField("l1", JField("l2", JField("l3", JField("num", 42)))) - val expected = JObject(List(("l1", JObject(List(("l2", JObject(List(("l3", JObject(List( - ("str", JString("hi")), - ("num", JInt(42)) - ))))))))))) - - MailchimpAdapter.mergeJFields(List(a, b)) mustEqual expected + val a = ("l1", Json.obj(("l2", Json.obj(("l3", Json.obj(("str", Json.fromString("hi")))))))) + val b = ("l1", Json.obj(("l2", Json.obj(("l3", Json.obj(("num", Json.fromInt(42)))))))) + val expected = json"""{ + "l1": { + "l2": { + "l3": { + "str": "hi", + "num": 42 + } + } + } + }""" + MailchimpAdapter.mergeJsons(List(a, b)) mustEqual expected } def e5 = - "SPEC NAME" || "PARAMS" | "EXPECTED OUTPUT" | - "Return Updated Params" !! Map("type" -> "subscribe", "fired_at" -> "2014-10-22 13:50:00") ! Map("type" -> "subscribe", "fired_at" -> "2014-10-22T13:50:00.000Z") | - "Return Same Params" !! Map("type" -> "subscribe", "id" -> "some_id") ! Map("type" -> "subscribe", "id" -> "some_id") |> { - (_, params, expected) => + "SPEC NAME" || "PARAMS" | "EXPECTED OUTPUT" | + "Return Updated Params" !! Map("type" -> "subscribe", "fired_at" -> "2014-10-22 13:50:00") ! Map( + "type" -> "subscribe", + "fired_at" -> "2014-10-22T13:50:00.000Z") | + "Return Same Params" !! Map("type" -> "subscribe", "id" -> "some_id") ! Map( + "type" -> "subscribe", + "id" -> "some_id") |> { (_, params, expected) => val actual = MailchimpAdapter.reformatParameters(params) actual mustEqual expected - } + } def e6 = { val body = "type=subscribe&data%5Bmerges%5D%5BLNAME%5D=Beemster" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", |"data":{ |"schema":"iglu:com.mailchimp/subscribe/jsonschema/1-0-0", |"data":{ - |"type":"subscribe", |"data":{ |"merges":{ |"LNAME":"Beemster" |} - |} + |}, + |"type":"subscribe" |} |} - |}""".stripMargin.replaceAll("[\n\r]","") + |}""".stripMargin.replaceAll("[\n\r]", "") val actual = MailchimpAdapter.toRawEvents(payload) - actual must beSuccessful(NonEmptyList(RawEvent(Shared.api, Map("tv" -> "com.mailchimp-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), ContentType.some, Shared.cljSource, Shared.context))) + actual must beSuccessful( + NonEmptyList( + RawEvent( + Shared.api, + Map("tv" -> "com.mailchimp-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context))) } def e7 = { val body = "type=subscribe&data%5Bmerges%5D%5BFNAME%5D=Agent&data%5Bmerges%5D%5BLNAME%5D=Smith" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", |"data":{ |"schema":"iglu:com.mailchimp/subscribe/jsonschema/1-0-0", |"data":{ - |"type":"subscribe", |"data":{ |"merges":{ - |"FNAME":"Agent", - |"LNAME":"Smith" + |"LNAME":"Smith", + |"FNAME":"Agent" |} - |} + |}, + |"type":"subscribe" |} |} - |}""".stripMargin.replaceAll("[\n\r]","") + |}""".stripMargin.replaceAll("[\n\r]", "") val actual = MailchimpAdapter.toRawEvents(payload) - actual must beSuccessful(NonEmptyList(RawEvent(Shared.api, Map("tv" -> "com.mailchimp-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), ContentType.some, Shared.cljSource, Shared.context))) + actual must beSuccessful( + NonEmptyList( + RawEvent( + Shared.api, + Map("tv" -> "com.mailchimp-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context))) } def e8 = - "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED SCHEMA" | - "Valid, type subscribe" !! "subscribe" ! "iglu:com.mailchimp/subscribe/jsonschema/1-0-0" | - "Valid, type unsubscribe" !! "unsubscribe" ! "iglu:com.mailchimp/unsubscribe/jsonschema/1-0-0" | - "Valid, type profile" !! "profile" ! "iglu:com.mailchimp/profile_update/jsonschema/1-0-0" | - "Valid, type email" !! "upemail" ! "iglu:com.mailchimp/email_address_change/jsonschema/1-0-0" | - "Valid, type cleaned" !! "cleaned" ! "iglu:com.mailchimp/cleaned_email/jsonschema/1-0-0" | - "Valid, type campaign" !! "campaign" ! "iglu:com.mailchimp/campaign_sending_status/jsonschema/1-0-0" |> { + "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED SCHEMA" | + "Valid, type subscribe" !! "subscribe" ! "iglu:com.mailchimp/subscribe/jsonschema/1-0-0" | + "Valid, type unsubscribe" !! "unsubscribe" ! "iglu:com.mailchimp/unsubscribe/jsonschema/1-0-0" | + "Valid, type profile" !! "profile" ! "iglu:com.mailchimp/profile_update/jsonschema/1-0-0" | + "Valid, type email" !! "upemail" ! "iglu:com.mailchimp/email_address_change/jsonschema/1-0-0" | + "Valid, type cleaned" !! "cleaned" ! "iglu:com.mailchimp/cleaned_email/jsonschema/1-0-0" | + "Valid, type campaign" !! "campaign" ! "iglu:com.mailchimp/campaign_sending_status/jsonschema/1-0-0" |> { (_, schema, expected) => - val body = "type="+schema - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) - val expectedJson = "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\""+expected+"\",\"data\":{\"type\":\""+schema+"\"}}}" + val body = "type=" + schema + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) + val expectedJson = "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"" + expected + "\",\"data\":{\"type\":\"" + schema + "\"}}}" val actual = MailchimpAdapter.toRawEvents(payload) - actual must beSuccessful(NonEmptyList(RawEvent(Shared.api, Map("tv" -> "com.mailchimp-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), ContentType.some, Shared.cljSource, Shared.context))) - } + actual must beSuccessful( + NonEmptyList( + RawEvent( + Shared.api, + Map("tv" -> "com.mailchimp-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context))) + } def e9 = - "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED OUTPUT" | - "Invalid, bad type" !! "bad" ! "MailChimp event failed: type parameter [bad] not recognized" | - "Invalid, no type" !! "" ! "MailChimp event failed: type parameter is empty - cannot determine event type" |> { + "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED OUTPUT" | + "Invalid, bad type" !! "bad" ! "MailChimp event failed: type parameter [bad] not recognized" | + "Invalid, no type" !! "" ! "MailChimp event failed: type parameter is empty - cannot determine event type" |> { (_, schema, expected) => - val body = "type="+schema - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val body = "type=" + schema + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val actual = MailchimpAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList(expected)) - } + } def e10 = { - val body = "type=unsubscribe&fired_at=2014-10-22+13%3A10%3A40&data%5Baction%5D=unsub&data%5Breason%5D=manual&data%5Bid%5D=94826aa750&data%5Bemail%5D=josh%40snowplowanalytics.com&data%5Bemail_type%5D=html&data%5Bip_opt%5D=82.225.169.220&data%5Bweb_id%5D=203740265&data%5Bmerges%5D%5BEMAIL%5D=josh%40snowplowanalytics.com&data%5Bmerges%5D%5BFNAME%5D=Joshua&data%5Bmerges%5D%5BLNAME%5D=Beemster&data%5Blist_id%5D=f1243a3b12" + val body = + "type=unsubscribe&fired_at=2014-10-22+13%3A10%3A40&data%5Baction%5D=unsub&data%5Breason%5D=manual&data%5Bid%5D=94826aa750&data%5Bemail%5D=josh%40snowplowanalytics.com&data%5Bemail_type%5D=html&data%5Bip_opt%5D=82.225.169.220&data%5Bweb_id%5D=203740265&data%5Bmerges%5D%5BEMAIL%5D=josh%40snowplowanalytics.com&data%5Bmerges%5D%5BFNAME%5D=Joshua&data%5Bmerges%5D%5BLNAME%5D=Beemster&data%5Blist_id%5D=f1243a3b12" val qs = toNameValuePairs("nuid" -> "123") - val payload = CollectorPayload(Shared.api, qs, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + qs, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -187,52 +257,81 @@ class MailchimpAdapterSpec extends Specification with DataTables with Validation |"schema":"iglu:com.mailchimp/unsubscribe/jsonschema/1-0-0", |"data":{ |"data":{ - |"ip_opt":"82.225.169.220", |"merges":{ - |"LNAME":"Beemster", + |"EMAIL":"josh@snowplowanalytics.com", |"FNAME":"Joshua", - |"EMAIL":"josh@snowplowanalytics.com" + |"LNAME":"Beemster" |}, - |"email":"josh@snowplowanalytics.com", - |"list_id":"f1243a3b12", - |"email_type":"html", - |"reason":"manual", - |"id":"94826aa750", + |"web_id":"203740265", |"action":"unsub", - |"web_id":"203740265" + |"id":"94826aa750", + |"reason":"manual", + |"email_type":"html", + |"list_id":"f1243a3b12", + |"email":"josh@snowplowanalytics.com", + |"ip_opt":"82.225.169.220" |}, - |"fired_at":"2014-10-22T13:10:40.000Z", - |"type":"unsubscribe" + |"type":"unsubscribe", + |"fired_at":"2014-10-22T13:10:40.000Z" |} |} - |}""".stripMargin.replaceAll("[\n\r]","") + |}""".stripMargin.replaceAll("[\n\r]", "") val actual = MailchimpAdapter.toRawEvents(payload) - actual must beSuccessful(NonEmptyList(RawEvent(Shared.api, Map("tv" -> "com.mailchimp-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson, "nuid" -> "123"), ContentType.some, Shared.cljSource, Shared.context))) + actual must beSuccessful( + NonEmptyList( + RawEvent( + Shared.api, + Map( + "tv" -> "com.mailchimp-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> expectedJson, + "nuid" -> "123"), + ContentType.some, + Shared.cljSource, + Shared.context))) } def e11 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) val actual = MailchimpAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList("Request body is empty: no MailChimp event to process")) } def e12 = { - val payload = CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) val actual = MailchimpAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Request body provided but content type empty, expected application/x-www-form-urlencoded for MailChimp")) + actual must beFailing(NonEmptyList( + "Request body provided but content type empty, expected application/x-www-form-urlencoded for MailChimp")) } def e13 = { - val payload = CollectorPayload(Shared.api, Nil, "application/json".some, "stub".some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + "application/json".some, + "stub".some, + Shared.cljSource, + Shared.context) val actual = MailchimpAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Content type of application/json provided, expected application/x-www-form-urlencoded for MailChimp")) + actual must beFailing(NonEmptyList( + "Content type of application/json provided, expected application/x-www-form-urlencoded for MailChimp")) } def e14 = { val body = "fired_at=2014-10-22+13%3A10%3A40" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val actual = MailchimpAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("No MailChimp type parameter provided: cannot determine event type")) + actual must beFailing( + NonEmptyList("No MailChimp type parameter provided: cannot determine event type")) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala index ee5586fdc..37886ec88 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala @@ -23,7 +23,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} -class MailgunAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class MailgunAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the MailgunAdapter functionality toRawEvents must return a Success Nel if every event 'delivered' in the payload is successful $e1 @@ -46,14 +50,15 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.mailgun", "v1") + val api = CollectorApi("com.mailgun", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/x-www-form-urlencoded" @@ -61,7 +66,13 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa def e1 = { val body = "X-Mailgun-Sid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&domain=sandboxbcd3ccb1a529415db665622619a61616.mailgun.org&message-headers=%5B%5B%22Sender%22%2C+%22postmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%22%5D%2C+%5B%22Date%22%2C+%22Mon%2C+27+Jun+2016+15%3A19%3A02+%2B0000%22%5D%2C+%5B%22X-Mailgun-Sid%22%2C+%22WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D%22%5D%2C+%5B%22Received%22%2C+%22by+luna.mailgun.net+with+HTTP%3B+Mon%2C+27+Jun+2016+15%3A19%3A01+%2B0000%22%5D%2C+%5B%22Message-Id%22%2C+%22%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22To%22%2C+%22Ronny+%3Ctest%40snowplowanalytics.com%3E%22%5D%2C+%5B%22From%22%2C+%22Mailgun+Sandbox+%3Cpostmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22Subject%22%2C+%22Hello+Ronny%22%5D%2C+%5B%22Content-Type%22%2C+%5B%22text%2Fplain%22%2C+%7B%22charset%22%3A+%22ascii%22%7D%5D%5D%2C+%5B%22Mime-Version%22%2C+%221.0%22%5D%2C+%5B%22Content-Transfer-Encoding%22%2C+%5B%227bit%22%2C+%7B%7D%5D%5D%5D&Message-Id=%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E&recipient=test%40snowplowanalytics.com&event=delivered×tamp=1467040750&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e&body-plain=" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -69,30 +80,37 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa |"schema":"iglu:com.mailgun/message_delivered/jsonschema/1-0-0", |"data":{ |"recipient":"test@snowplowanalytics.com", - |"messageHeaders":"[[\"Sender\", \"postmaster@sandboxbcd3ccb1a529415db665622619a61616.mailgun.org\"], [\"Date\", \"Mon, 27 Jun 2016 15:19:02 +0000\"], [\"X-Mailgun-Sid\", \"WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0=\"], [\"Received\", \"by luna.mailgun.net with HTTP; Mon, 27 Jun 2016 15:19:01 +0000\"], [\"Message-Id\", \"<20160627151901.3295.78981.1336C636@sandboxbcd3ccb1a529415db665622619a61616.mailgun.org>\"], [\"To\", \"Ronny \"], [\"From\", \"Mailgun Sandbox \"], [\"Subject\", \"Hello Ronny\"], [\"Content-Type\", [\"text/plain\", {\"charset\": \"ascii\"}]], [\"Mime-Version\", \"1.0\"], [\"Content-Transfer-Encoding\", [\"7bit\", {}]]]", |"timestamp":"2016-06-27T15:19:10.000Z", + |"xMailgunSid":"WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0=", |"domain":"sandboxbcd3ccb1a529415db665622619a61616.mailgun.org", |"signature":"9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e", + |"messageHeaders":"[[\"Sender\", \"postmaster@sandboxbcd3ccb1a529415db665622619a61616.mailgun.org\"], [\"Date\", \"Mon, 27 Jun 2016 15:19:02 +0000\"], [\"X-Mailgun-Sid\", \"WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0=\"], [\"Received\", \"by luna.mailgun.net with HTTP; Mon, 27 Jun 2016 15:19:01 +0000\"], [\"Message-Id\", \"<20160627151901.3295.78981.1336C636@sandboxbcd3ccb1a529415db665622619a61616.mailgun.org>\"], [\"To\", \"Ronny \"], [\"From\", \"Mailgun Sandbox \"], [\"Subject\", \"Hello Ronny\"], [\"Content-Type\", [\"text/plain\", {\"charset\": \"ascii\"}]], [\"Mime-Version\", \"1.0\"], [\"Content-Transfer-Encoding\", [\"7bit\", {}]]]", |"token":"c2fc6a36198fa651243afb6042867b7490e480843198008c6b", - |"messageId":"<20160627151901.3295.78981.1336C636@sandboxbcd3ccb1a529415db665622619a61616.mailgun.org>", - |"xMailgunSid":"WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0=" + |"messageId":"<20160627151901.3295.78981.1336C636@sandboxbcd3ccb1a529415db665622619a61616.mailgun.org>" |} |} |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) MailgunAdapter.toRawEvents(payload) must beSuccessful(expected) } def e2 = { val body = "city=San+Francisco&domain=sandboxbcd3ccb1a529415db665622619a61616.mailgun.org&device-type=desktop&my_var_1=Mailgun+Variable+%231&country=US®ion=CA&client-name=Chrome&user-agent=Mozilla%2F5.0+%28X11%3B+Linux+x86_64%29+AppleWebKit%2F537.31+%28KHTML%2C+like+Gecko%29+Chrome%2F26.0.1410.43+Safari%2F537.31&client-os=Linux&my_var_2=awesome&ip=50.56.129.169&client-type=browser&recipient=alice%40example.com&event=opened×tamp=1467297128&token=c2eecf923f9820812338de117346d6448ea2cf7e2e98cfa1a0&signature=9c70b687ef784ec5ed78f4d9442d641a9cfc7b909f9bf43d9ce7e44b3448cf97&body-plain=" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -102,37 +120,43 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa |"recipient":"alice@example.com", |"city":"San Francisco", |"ip":"50.56.129.169", - |"myVar2":"awesome", |"timestamp":"2016-06-30T14:32:08.000Z", - |"userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31", |"domain":"sandboxbcd3ccb1a529415db665622619a61616.mailgun.org", |"signature":"9c70b687ef784ec5ed78f4d9442d641a9cfc7b909f9bf43d9ce7e44b3448cf97", + |"deviceType":"desktop", |"country":"US", + |"userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31", + |"myVar1":"Mailgun Variable #1", |"clientType":"browser", - |"clientOs":"Linux", |"token":"c2eecf923f9820812338de117346d6448ea2cf7e2e98cfa1a0", - |"clientName":"Chrome", |"region":"CA", - |"deviceType":"desktop", - |"myVar1":"Mailgun Variable #1" - + |"clientName":"Chrome", + |"myVar2":"awesome", + |"clientOs":"Linux" |} |} |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) MailgunAdapter.toRawEvents(payload) must beSuccessful(expected) } def e3 = { val body = "city=San+Francisco&domain=sandboxbcd3ccb1a529415db665622619a61616.mailgun.org&device-type=desktop&my_var_1=Mailgun+Variable+%231&country=US®ion=CA&client-name=Chrome&user-agent=Mozilla%2F5.0+%28X11%3B+Linux+x86_64%29+AppleWebKit%2F537.31+%28KHTML%2C+like+Gecko%29+Chrome%2F26.0.1410.43+Safari%2F537.31&client-os=Linux&my_var_2=awesome&url=http%3A%2F%2Fmailgun.net&ip=50.56.129.169&client-type=browser&recipient=alice%40example.com&event=clicked×tamp=1467297069&token=cd89cd860be0e318371f4220b7e0f368b60ac9ab066354737f&signature=ffe2d315a1d937bd09d9f5c35ddac1eb448818e2203f5a41e3a7bd1fb47da385&body-plain=" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -142,37 +166,44 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa |"recipient":"alice@example.com", |"city":"San Francisco", |"ip":"50.56.129.169", - |"myVar2":"awesome", |"timestamp":"2016-06-30T14:31:09.000Z", |"url":"http://mailgun.net", - |"userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31", |"domain":"sandboxbcd3ccb1a529415db665622619a61616.mailgun.org", |"signature":"ffe2d315a1d937bd09d9f5c35ddac1eb448818e2203f5a41e3a7bd1fb47da385", + |"deviceType":"desktop", |"country":"US", + |"userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31", + |"myVar1":"Mailgun Variable #1", |"clientType":"browser", - |"clientOs":"Linux", |"token":"cd89cd860be0e318371f4220b7e0f368b60ac9ab066354737f", - |"clientName":"Chrome", |"region":"CA", - |"deviceType":"desktop", - |"myVar1":"Mailgun Variable #1" + |"clientName":"Chrome", + |"myVar2":"awesome", + |"clientOs":"Linux" |} |} |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) MailgunAdapter.toRawEvents(payload) must beSuccessful(expected) } def e4 = { val body = "ip=50.56.129.169&city=San+Francisco&domain=sandboxbcd3ccb1a529415db665622619a61616.mailgun.org&device-type=desktop&my_var_1=Mailgun+Variable+%231&country=US®ion=CA&client-name=Chrome&user-agent=Mozilla%2F5.0+%28X11%3B+Linux+x86_64%29+AppleWebKit%2F537.31+%28KHTML%2C+like+Gecko%29+Chrome%2F26.0.1410.43+Safari%2F537.31&client-os=Linux&my_var_2=awesome&client-type=browser&tag=%2A&recipient=alice%40example.com&event=unsubscribed×tamp=1467297059&token=45272007729d82a7f7471d17e21298ee1a3899df65ba4a63ff&signature=150f32facb18c47273cf890d4aa13354ea789ad7b076554e8b324be6f446e2ad&body-plain=" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -182,65 +213,67 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa |"recipient":"alice@example.com", |"city":"San Francisco", |"ip":"50.56.129.169", - |"myVar2":"awesome", |"timestamp":"2016-06-30T14:30:59.000Z", |"tag":"*", - |"userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31", |"domain":"sandboxbcd3ccb1a529415db665622619a61616.mailgun.org", |"signature":"150f32facb18c47273cf890d4aa13354ea789ad7b076554e8b324be6f446e2ad", + |"deviceType":"desktop", |"country":"US", + |"userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31", + |"myVar1":"Mailgun Variable #1", |"clientType":"browser", - |"clientOs":"Linux", |"token":"45272007729d82a7f7471d17e21298ee1a3899df65ba4a63ff", - |"clientName":"Chrome", |"region":"CA", - |"deviceType":"desktop", - |"myVar1":"Mailgun Variable #1" + |"clientName":"Chrome", + |"myVar2":"awesome", + |"clientOs":"Linux" |} |} |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.mailgun-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) MailgunAdapter.toRawEvents(payload) must beSuccessful(expected) } def e5 = { val body = "--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"Message-Id\"\n\n<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"X-Mailgun-Sid\"\n\nWyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"attachment-count\"\n\n1\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"body-plain\"\n\n\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"code\"\n\n605\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"description\"\n\nNot delivering to previously bounced address\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"domain\"\n\nsandbox57070072075d4cfd9008d4332108734c.mailgun.org\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"event\"\n\ndropped\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"message-headers\"\n\n[[\"Received\", \"by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013 19:26:59 +0000\"], [\"Content-Type\", [\"multipart/alternative\", {\"boundary\": \"23041bcdfae54aafb801a8da0283af85\"}]], [\"Mime-Version\", \"1.0\"], [\"Subject\", \"Test drop webhook\"], [\"From\", \"Bob \"], [\"To\", \"Alice \"], [\"Message-Id\", \"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\"], [\"List-Unsubscribe\", \"\"], [\"X-Mailgun-Sid\", \"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\"], [\"X-Mailgun-Variables\", \"{\\\"my_var_1\\\": \\\"Mailgun Variable #1\\\", \\\"my_var_2\\\": \\\"awesome\\\"}\"], [\"Date\", \"Fri, 03 May 2013 19:26:59 +0000\"], [\"Sender\", \"bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\"]]\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"my_var_2\"\n\nawesome\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"my_var_1\"\n\nMailgun Variable #1\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"reason\"\n\nhardfail\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"recipient\"\n\nalice@example.com\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"signature\"\n\n71f812485ae3fb398de8d1a86b139f24391d604fd94dab59e7c99cfcd506885c\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"timestamp\"\n\n1510161862\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"token\"\n\n9e3fffc7eba57e282e89f7afcf243563868e9de4ecfea78c09\n--353d603f-eede-4b49-97ac-724fbc54ea3c\nContent-Disposition: form-data; name=\"attachment-1\"; filename=\"message.mime\"\nContent-Type: application/octet-stream\nContent-Length: 1386\n\nReceived: by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013\n 19:26:59 +0000\nContent-Type: multipart/alternative; boundary=\"23041bcdfae54aafb801a8da0283af85\"\nMime-Version: 1.0\nSubject: Test drop webhook\nFrom: Bob \nTo: Alice \nMessage-Id: <20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\nList-Unsubscribe: \nX-Mailgun-Sid: WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\nX-Mailgun-Variables: {\"my_var_1\": \"Mailgun Variable #1\", \"my_var_2\": \"awesome\"}\nDate: Fri, 03 May 2013 19:26:59 +0000\nSender: bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/plain; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\nHi Alice, I sent an email to this address but it was bounced.\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/html; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\n\n Hi Alice, I sent an email to this address but it was bounced.\n
\n\n--23041bcdfae54aafb801a8da0283af85--\n\n--353d603f-eede-4b49-97ac-724fbc54ea3c--" - val payload = CollectorPayload(Shared.api, - Nil, - Some("multipart/form-data; boundary=353d603f-eede-4b49-97ac-724fbc54ea3c"), - body.some, - Shared.cljSource, - Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + Some("multipart/form-data; boundary=353d603f-eede-4b49-97ac-724fbc54ea3c"), + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", - |"data":{ - |"schema":"iglu:com.mailgun/message_dropped/jsonschema/1-0-0", - |"data":{ - |"attachmentCount":1, - |"recipient":"alice@example.com", - |"messageHeaders":"[[\"Received\", \"by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013 19:26:59 +0000\"], [\"Content-Type\", [\"multipart/alternative\", {\"boundary\": \"23041bcdfae54aafb801a8da0283af85\"}]], [\"Mime-Version\", \"1.0\"], [\"Subject\", \"Test drop webhook\"], [\"From\", \"Bob \"], [\"To\", \"Alice \"], [\"Message-Id\", \"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\"], [\"List-Unsubscribe\", \"\"], [\"X-Mailgun-Sid\", \"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\"], [\"X-Mailgun-Variables\", \"{\\\"my_var_1\\\": \\\"Mailgun Variable #1\\\", \\\"my_var_2\\\": \\\"awesome\\\"}\"], [\"Date\", \"Fri, 03 May 2013 19:26:59 +0000\"], [\"Sender\", \"bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\"]]", - |"myVar2":"awesome", - |"timestamp":"2017-11-08T17:24:22.000Z", - |"description":"Not delivering to previously bounced address", - |"domain":"sandbox57070072075d4cfd9008d4332108734c.mailgun.org", - |"signature":"71f812485ae3fb398de8d1a86b139f24391d604fd94dab59e7c99cfcd506885c", - |"reason":"hardfail", - |"code":"605", - |"token":"9e3fffc7eba57e282e89f7afcf243563868e9de4ecfea78c09", - |"messageId":"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>", - |"xMailgunSid":"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=", - |"myVar1":"Mailgun Variable #1", - |"attachment1":"Received: by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013\n 19:26:59 +0000\nContent-Type: multipart/alternative; boundary=\"23041bcdfae54aafb801a8da0283af85\"\nMime-Version: 1.0\nSubject: Test drop webhook\nFrom: Bob \nTo: Alice \nMessage-Id: <20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\nList-Unsubscribe: \nX-Mailgun-Sid: WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\nX-Mailgun-Variables: {\"my_var_1\": \"Mailgun Variable #1\", \"my_var_2\": \"awesome\"}\nDate: Fri, 03 May 2013 19:26:59 +0000\nSender: bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/plain; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\nHi Alice, I sent an email to this address but it was bounced.\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/html; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\n\n Hi Alice, I sent an email to this address but it was bounced.\n
\n\n--23041bcdfae54aafb801a8da0283af85--" - |} - |} + |"data":{ + |"schema":"iglu:com.mailgun/message_dropped/jsonschema/1-0-0", + |"data":{ + |"recipient":"alice@example.com", + |"timestamp":"2017-11-08T17:24:22.000Z", + |"xMailgunSid":"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=", + |"description":"Not delivering to previously bounced address", + |"domain":"sandbox57070072075d4cfd9008d4332108734c.mailgun.org", + |"signature":"71f812485ae3fb398de8d1a86b139f24391d604fd94dab59e7c99cfcd506885c", + |"reason":"hardfail", + |"messageHeaders":"[[\"Received\", \"by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013 19:26:59 +0000\"], [\"Content-Type\", [\"multipart/alternative\", {\"boundary\": \"23041bcdfae54aafb801a8da0283af85\"}]], [\"Mime-Version\", \"1.0\"], [\"Subject\", \"Test drop webhook\"], [\"From\", \"Bob \"], [\"To\", \"Alice \"], [\"Message-Id\", \"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\"], [\"List-Unsubscribe\", \"\"], [\"X-Mailgun-Sid\", \"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\"], [\"X-Mailgun-Variables\", \"{\\\"my_var_1\\\": \\\"Mailgun Variable #1\\\", \\\"my_var_2\\\": \\\"awesome\\\"}\"], [\"Date\", \"Fri, 03 May 2013 19:26:59 +0000\"], [\"Sender\", \"bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\"]]", + |"code":"605", + |"myVar1":"Mailgun Variable #1", + |"attachment1":"Received: by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013\n 19:26:59 +0000\nContent-Type: multipart/alternative; boundary=\"23041bcdfae54aafb801a8da0283af85\"\nMime-Version: 1.0\nSubject: Test drop webhook\nFrom: Bob \nTo: Alice \nMessage-Id: <20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\nList-Unsubscribe: \nX-Mailgun-Sid: WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\nX-Mailgun-Variables: {\"my_var_1\": \"Mailgun Variable #1\", \"my_var_2\": \"awesome\"}\nDate: Fri, 03 May 2013 19:26:59 +0000\nSender: bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/plain; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\nHi Alice, I sent an email to this address but it was bounced.\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/html; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\n\n Hi Alice, I sent an email to this address but it was bounced.\n
\n\n--23041bcdfae54aafb801a8da0283af85--", + |"token":"9e3fffc7eba57e282e89f7afcf243563868e9de4ecfea78c09", + |"messageId":"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>", + |"attachmentCount":1, + |"myVar2":"awesome" + |} + |} |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( RawEvent( @@ -253,29 +286,38 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa MailgunAdapter.toRawEvents(payload) must beSuccessful(expected) } def e6 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) MailgunAdapter.toRawEvents(payload) must beFailing( NonEmptyList("Request body is empty: no Mailgun events to process")) } def e7 = { - val body = "" - val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) + val body = "" + val payload = + CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) MailgunAdapter.toRawEvents(payload) must beFailing(NonEmptyList( "Request body provided but content type empty, expected application/x-www-form-urlencoded or multipart/form-data for Mailgun")) } def e8 = { - val body = "" - val ct = "application/json" - val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) + val body = "" + val ct = "application/json" + val payload = + CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) MailgunAdapter.toRawEvents(payload) must beFailing(NonEmptyList( "Content type of application/json provided, expected application/x-www-form-urlencoded or multipart/form-data for Mailgun")) } def e9 = { - val body = "" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val body = "" + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Mailgun event body is empty: nothing to process") MailgunAdapter.toRawEvents(payload) must beFailing(expected) } @@ -283,16 +325,28 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa def e10 = { val body = "X-MailgunSid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&event=delivered×tamp=1467040750&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e&recipient=<>" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( - "MailgunAdapter could not parse body: [Illegal character in query at index 261: http://localhost/?X-MailgunSid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&event=delivered×tamp=1467040750&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e&recipient=<>]") + "Mailgun adapter could not parse body: [Illegal character in query at index 261: http://localhost/?X-MailgunSid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&event=delivered×tamp=1467040750&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e&recipient=<>]") MailgunAdapter.toRawEvents(payload) must beFailing(expected) } def e11 = { val body = "X-MailgunSid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D×tamp=1467040750&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("No Mailgun event parameter provided: cannot determine event type") MailgunAdapter.toRawEvents(payload) must beFailing(expected) } @@ -300,7 +354,13 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa def e12 = { val body = "X-MailgunSid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&event=released×tamp=1467040750&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Mailgun event failed: type parameter [released] not recognized") MailgunAdapter.toRawEvents(payload) must beFailing(expected) } @@ -308,7 +368,13 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa def e13 = { val body = "X-Mailgun-Sid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&domain=sandboxbcd3ccb1a529415db665622619a61616.mailgun.org&message-headers=%5B%5B%22Sender%22%2C+%22postmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%22%5D%2C+%5B%22Date%22%2C+%22Mon%2C+27+Jun+2016+15%3A19%3A02+%2B0000%22%5D%2C+%5B%22X-Mailgun-Sid%22%2C+%22WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D%22%5D%2C+%5B%22Received%22%2C+%22by+luna.mailgun.net+with+HTTP%3B+Mon%2C+27+Jun+2016+15%3A19%3A01+%2B0000%22%5D%2C+%5B%22Message-Id%22%2C+%22%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22To%22%2C+%22Ronny+%3Ctest%40snowplowanalytics.com%3E%22%5D%2C+%5B%22From%22%2C+%22Mailgun+Sandbox+%3Cpostmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22Subject%22%2C+%22Hello+Ronny%22%5D%2C+%5B%22Content-Type%22%2C+%5B%22text%2Fplain%22%2C+%7B%22charset%22%3A+%22ascii%22%7D%5D%5D%2C+%5B%22Mime-Version%22%2C+%221.0%22%5D%2C+%5B%22Content-Transfer-Encoding%22%2C+%5B%227bit%22%2C+%7B%7D%5D%5D%5D&Message-Id=%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E&recipient=test%40snowplowanalytics.com&event=delivered&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e&body-plain=" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Mailgun event data missing 'timestamp'") MailgunAdapter.toRawEvents(payload) must beFailing(expected) } @@ -316,7 +382,13 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa def e14 = { val body = "X-Mailgun-Sid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&domain=sandboxbcd3ccb1a529415db665622619a61616.mailgun.org&message-headers=%5B%5B%22Sender%22%2C+%22postmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%22%5D%2C+%5B%22Date%22%2C+%22Mon%2C+27+Jun+2016+15%3A19%3A02+%2B0000%22%5D%2C+%5B%22X-Mailgun-Sid%22%2C+%22WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D%22%5D%2C+%5B%22Received%22%2C+%22by+luna.mailgun.net+with+HTTP%3B+Mon%2C+27+Jun+2016+15%3A19%3A01+%2B0000%22%5D%2C+%5B%22Message-Id%22%2C+%22%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22To%22%2C+%22Ronny+%3Ctest%40snowplowanalytics.com%3E%22%5D%2C+%5B%22From%22%2C+%22Mailgun+Sandbox+%3Cpostmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22Subject%22%2C+%22Hello+Ronny%22%5D%2C+%5B%22Content-Type%22%2C+%5B%22text%2Fplain%22%2C+%7B%22charset%22%3A+%22ascii%22%7D%5D%5D%2C+%5B%22Mime-Version%22%2C+%221.0%22%5D%2C+%5B%22Content-Transfer-Encoding%22%2C+%5B%227bit%22%2C+%7B%7D%5D%5D%5D&Message-Id=%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E&recipient=test%40snowplowanalytics.com&event=delivered×tamp=1467040750&signature=9387fb0e5ff02de5e159594173f02c95c55d7e681b40a7b930ed4d0a3cbbdd6e&body-plain=" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Mailgun event data missing 'token'") MailgunAdapter.toRawEvents(payload) must beFailing(expected) } @@ -324,7 +396,13 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidationMa def e15 = { val body = "X-Mailgun-Sid=WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D&domain=sandboxbcd3ccb1a529415db665622619a61616.mailgun.org&message-headers=%5B%5B%22Sender%22%2C+%22postmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%22%5D%2C+%5B%22Date%22%2C+%22Mon%2C+27+Jun+2016+15%3A19%3A02+%2B0000%22%5D%2C+%5B%22X-Mailgun-Sid%22%2C+%22WyIxZjQzMiIsICJyb25ueUBrZGUub3JnIiwgIjliMjYwIl0%3D%22%5D%2C+%5B%22Received%22%2C+%22by+luna.mailgun.net+with+HTTP%3B+Mon%2C+27+Jun+2016+15%3A19%3A01+%2B0000%22%5D%2C+%5B%22Message-Id%22%2C+%22%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22To%22%2C+%22Ronny+%3Ctest%40snowplowanalytics.com%3E%22%5D%2C+%5B%22From%22%2C+%22Mailgun+Sandbox+%3Cpostmaster%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E%22%5D%2C+%5B%22Subject%22%2C+%22Hello+Ronny%22%5D%2C+%5B%22Content-Type%22%2C+%5B%22text%2Fplain%22%2C+%7B%22charset%22%3A+%22ascii%22%7D%5D%5D%2C+%5B%22Mime-Version%22%2C+%221.0%22%5D%2C+%5B%22Content-Transfer-Encoding%22%2C+%5B%227bit%22%2C+%7B%7D%5D%5D%5D&Message-Id=%3C20160627151901.3295.78981.1336C636%40sandboxbcd3ccb1a529415db665622619a61616.mailgun.org%3E&recipient=test%40snowplowanalytics.com&event=delivered×tamp=1467040750&token=c2fc6a36198fa651243afb6042867b7490e480843198008c6b&body-plain=" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Mailgun event data missing 'signature'") MailgunAdapter.toRawEvents(payload) must beFailing(expected) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala index 7316fc41a..ceb44a541 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala @@ -14,17 +14,21 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry +import io.circe.literal._ import org.joda.time.DateTime import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} -class MandrillAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class MandrillAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the MandrillAdapter functionality payloadBodyToEvents must return a Success List[JValue] for a valid events string $e1 @@ -40,52 +44,58 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.mandrill", "v1") + val api = CollectorApi("com.mandrill", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/x-www-form-urlencoded" def e1 = { - val bodyStr = "mandrill_events=%5B%7B%22event%22%3A%20%22subscribe%22%7D%5D" - val expected = List(JObject(List(("event", JString("subscribe"))))) + val bodyStr = "mandrill_events=%5B%7B%22event%22%3A%20%22subscribe%22%7D%5D" + val expected = List(json"""{"event": "subscribe"}""") MandrillAdapter.payloadBodyToEvents(bodyStr) must beSuccessful(expected) } def e2 = - "SPEC NAME" || "STRING TO PROCESS" | "EXPECTED OUTPUT" | - "Failure, empty events string" !! "mandrill_events=" ! "Mandrill events string is empty: nothing to process" | + "SPEC NAME" || "STRING TO PROCESS" | "EXPECTED OUTPUT" | + "Failure, empty events string" !! "mandrill_events=" ! "Mandrill events string is empty: nothing to process" | "Failure, too many key-value pairs" !! "mandrill_events=some&mandrill_extra=some" ! "Mapped Mandrill body has invalid count of keys: 2" | - "Failure, incorrect key" !! "events_mandrill=something" ! "Mapped Mandrill body does not have 'mandrill_events' as a key" |> { + "Failure, incorrect key" !! "events_mandrill=something" ! "Mapped Mandrill body does not have 'mandrill_events' as a key" |> { (_, str, expected) => MandrillAdapter.payloadBodyToEvents(str) must beFailing(expected) } def e3 = { val bodyStr = "mandrill_events=%5B%7B%22event%22%3A%22click%7D%5D" - val expected = - "Mandrill events string failed to parse into JSON: [com.fasterxml.jackson.core.io.JsonEOFException: Unexpected end-of-input: was expecting closing quote for a string value at [Source: (String)\"[{\"event\":\"click}]\"; line: 1, column: 37]]" + val expected = "Mandrill events couldn't be parsed as JSON: [exhausted input]" MandrillAdapter.payloadBodyToEvents(bodyStr) must beFailing(expected) } def e4 = { // Spec for nine seperate events being passed and returned. val bodyStr = "mandrill_events=%5B%7B%22event%22%3A%22send%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%5D%2C%22clicks%22%3A%5B%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22deferral%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%5D%2C%22clicks%22%3A%5B%5D%2C%22state%22%3A%22deferred%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa1%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%2C%22smtp_events%22%3A%5B%7B%22destination_ip%22%3A%22127.0.0.1%22%2C%22diag%22%3A%22451+4.3.5+Temporarily+unavailable%2C+try+again+later.%22%2C%22source_ip%22%3A%22127.0.0.1%22%2C%22ts%22%3A1365111111%2C%22type%22%3A%22deferred%22%2C%22size%22%3A0%7D%5D%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa1%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22hard_bounce%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22state%22%3A%22bounced%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa2%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%2C%22bounce_description%22%3A%22bad_mailbox%22%2C%22bgtools_code%22%3A10%2C%22diag%22%3A%22smtp%3B550+5.1.1+The+email+account+that+you+tried+to+reach+does+not+exist.+Please+try+double-checking+the+recipient%27s+email+address+for+typos+or+unnecessary+spaces.%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa2%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22soft_bounce%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22state%22%3A%22soft-bounced%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa3%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%2C%22bounce_description%22%3A%22mailbox_full%22%2C%22bgtools_code%22%3A22%2C%22diag%22%3A%22smtp%3B552+5.2.2+Over+Quota%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa3%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22open%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa4%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa4%22%2C%22ip%22%3A%22127.0.0.1%22%2C%22location%22%3A%7B%22country_short%22%3A%22US%22%2C%22country%22%3A%22United+States%22%2C%22region%22%3A%22Oklahoma%22%2C%22city%22%3A%22Oklahoma+City%22%2C%22latitude%22%3A35.4675598145%2C%22longitude%22%3A-97.5164337158%2C%22postal_code%22%3A%2273101%22%2C%22timezone%22%3A%22-05%3A00%22%7D%2C%22user_agent%22%3A%22Mozilla%5C%2F5.0+%28Macintosh%3B+U%3B+Intel+Mac+OS+X+10.6%3B+en-US%3B+rv%3A1.9.1.8%29+Gecko%5C%2F20100317+Postbox%5C%2F1.1.3%22%2C%22user_agent_parsed%22%3A%7B%22type%22%3A%22Email+Client%22%2C%22ua_family%22%3A%22Postbox%22%2C%22ua_name%22%3A%22Postbox+1.1.3%22%2C%22ua_version%22%3A%221.1.3%22%2C%22ua_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_company%22%3A%22Postbox%2C+Inc.%22%2C%22ua_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fpostbox.png%22%2C%22os_family%22%3A%22OS+X%22%2C%22os_name%22%3A%22OS+X+10.6+Snow+Leopard%22%2C%22os_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2Fosx%5C%2F%22%2C%22os_company%22%3A%22Apple+Computer%2C+Inc.%22%2C%22os_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2F%22%2C%22os_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fmacosx.png%22%2C%22mobile%22%3Afalse%7D%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22click%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa5%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa5%22%2C%22ip%22%3A%22127.0.0.1%22%2C%22location%22%3A%7B%22country_short%22%3A%22US%22%2C%22country%22%3A%22United+States%22%2C%22region%22%3A%22Oklahoma%22%2C%22city%22%3A%22Oklahoma+City%22%2C%22latitude%22%3A35.4675598145%2C%22longitude%22%3A-97.5164337158%2C%22postal_code%22%3A%2273101%22%2C%22timezone%22%3A%22-05%3A00%22%7D%2C%22user_agent%22%3A%22Mozilla%5C%2F5.0+%28Macintosh%3B+U%3B+Intel+Mac+OS+X+10.6%3B+en-US%3B+rv%3A1.9.1.8%29+Gecko%5C%2F20100317+Postbox%5C%2F1.1.3%22%2C%22user_agent_parsed%22%3A%7B%22type%22%3A%22Email+Client%22%2C%22ua_family%22%3A%22Postbox%22%2C%22ua_name%22%3A%22Postbox+1.1.3%22%2C%22ua_version%22%3A%221.1.3%22%2C%22ua_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_company%22%3A%22Postbox%2C+Inc.%22%2C%22ua_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fpostbox.png%22%2C%22os_family%22%3A%22OS+X%22%2C%22os_name%22%3A%22OS+X+10.6+Snow+Leopard%22%2C%22os_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2Fosx%5C%2F%22%2C%22os_company%22%3A%22Apple+Computer%2C+Inc.%22%2C%22os_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2F%22%2C%22os_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fmacosx.png%22%2C%22mobile%22%3Afalse%7D%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22spam%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa6%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa6%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22unsub%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa7%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa7%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22reject%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%5D%2C%22clicks%22%3A%5B%5D%2C%22state%22%3A%22rejected%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa8%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa8%22%2C%22ts%22%3A1415267366%7D%5D" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_sent/jsonschema/1-0-0","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"opens":[],"clicks":[],"state":"sent","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa","_version":"exampleaaaaaaaaaaaaaaa"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_sent/jsonschema/1-0-0","data":{"msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"sent","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa","tags":["webhook-example"],"ts":"2013-04-04T21:13:19.000Z","clicks":[],"metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","opens":[]},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa","ts":"2014-11-06T09:49:26.000Z"}}}""" ), ContentType.some, Shared.cljSource, @@ -94,10 +104,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_delayed/jsonschema/1-0-1","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"opens":[],"clicks":[],"state":"deferred","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa1","_version":"exampleaaaaaaaaaaaaaaa","smtp_events":[{"destination_ip":"127.0.0.1","diag":"451 4.3.5 Temporarily unavailable, try again later.","source_ip":"127.0.0.1","ts":"2013-04-04T21:31:51.000Z","type":"deferred","size":0}]},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa1","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_delayed/jsonschema/1-0-1","data":{"msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"deferred","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa1","tags":["webhook-example"],"ts":"2013-04-04T21:13:19.000Z","smtp_events":[{"size":0,"destination_ip":"127.0.0.1","diag":"451 4.3.5 Temporarily unavailable, try again later.","ts":"2013-04-04T21:31:51.000Z","source_ip":"127.0.0.1","type":"deferred"}],"clicks":[],"metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","opens":[]},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa1","ts":"2014-11-06T09:49:26.000Z"}}}""" ), ContentType.some, Shared.cljSource, @@ -106,10 +116,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_bounced/jsonschema/1-0-1","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"state":"bounced","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa2","_version":"exampleaaaaaaaaaaaaaaa","bounce_description":"bad_mailbox","bgtools_code":10,"diag":"smtp;550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces."},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa2","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_bounced/jsonschema/1-0-1","data":{"msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"bounced","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa2","tags":["webhook-example"],"diag":"smtp;550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces.","ts":"2013-04-04T21:13:19.000Z","metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","bounce_description":"bad_mailbox","bgtools_code":10},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa2","ts":"2014-11-06T09:49:26.000Z"}}}""" ), ContentType.some, Shared.cljSource, @@ -118,10 +128,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_soft_bounced/jsonschema/1-0-1","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"state":"soft-bounced","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa3","_version":"exampleaaaaaaaaaaaaaaa","bounce_description":"mailbox_full","bgtools_code":22,"diag":"smtp;552 5.2.2 Over Quota"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa3","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_soft_bounced/jsonschema/1-0-1","data":{"msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"soft-bounced","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa3","tags":["webhook-example"],"diag":"smtp;552 5.2.2 Over Quota","ts":"2013-04-04T21:13:19.000Z","metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","bounce_description":"mailbox_full","bgtools_code":22},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa3","ts":"2014-11-06T09:49:26.000Z"}}}""" ), ContentType.some, Shared.cljSource, @@ -130,10 +140,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_opened/jsonschema/1-0-1","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"opens":[{"ts":"2013-04-04T21:31:51.000Z"}],"clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"state":"sent","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa4","_version":"exampleaaaaaaaaaaaaaaa"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa4","ip":"127.0.0.1","location":{"country_short":"US","country":"United States","region":"Oklahoma","city":"Oklahoma City","latitude":35.4675598145,"longitude":-97.5164337158,"postal_code":"73101","timezone":"-05:00"},"user_agent":"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.8) Gecko/20100317 Postbox/1.1.3","user_agent_parsed":{"type":"Email Client","ua_family":"Postbox","ua_name":"Postbox 1.1.3","ua_version":"1.1.3","ua_url":"http://www.postbox-inc.com/","ua_company":"Postbox, Inc.","ua_company_url":"http://www.postbox-inc.com/","ua_icon":"http://cdn.mandrill.com/img/email-client-icons/postbox.png","os_family":"OS X","os_name":"OS X 10.6 Snow Leopard","os_url":"http://www.apple.com/osx/","os_company":"Apple Computer, Inc.","os_company_url":"http://www.apple.com/","os_icon":"http://cdn.mandrill.com/img/email-client-icons/macosx.png","mobile":false},"ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_opened/jsonschema/1-0-1","data":{"ip":"127.0.0.1","location":{"city":"Oklahoma City","latitude":35.4675598145,"timezone":"-05:00","country":"United States","longitude":-97.5164337158,"country_short":"US","postal_code":"73101","region":"Oklahoma"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa4","ts":"2014-11-06T09:49:26.000Z","user_agent":"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.8) Gecko/20100317 Postbox/1.1.3","msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"sent","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa4","tags":["webhook-example"],"ts":"2013-04-04T21:13:19.000Z","clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","opens":[{"ts":"2013-04-04T21:31:51.000Z"}]},"user_agent_parsed":{"os_company_url":"http://www.apple.com/","os_family":"OS X","os_company":"Apple Computer, Inc.","os_url":"http://www.apple.com/osx/","ua_url":"http://www.postbox-inc.com/","ua_icon":"http://cdn.mandrill.com/img/email-client-icons/postbox.png","ua_version":"1.1.3","os_name":"OS X 10.6 Snow Leopard","ua_company":"Postbox, Inc.","ua_family":"Postbox","os_icon":"http://cdn.mandrill.com/img/email-client-icons/macosx.png","ua_company_url":"http://www.postbox-inc.com/","ua_name":"Postbox 1.1.3","type":"Email Client","mobile":false}}}}""" ), ContentType.some, Shared.cljSource, @@ -142,10 +152,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_clicked/jsonschema/1-0-1","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"opens":[{"ts":"2013-04-04T21:31:51.000Z"}],"clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"state":"sent","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa5","_version":"exampleaaaaaaaaaaaaaaa"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa5","ip":"127.0.0.1","location":{"country_short":"US","country":"United States","region":"Oklahoma","city":"Oklahoma City","latitude":35.4675598145,"longitude":-97.5164337158,"postal_code":"73101","timezone":"-05:00"},"user_agent":"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.8) Gecko/20100317 Postbox/1.1.3","user_agent_parsed":{"type":"Email Client","ua_family":"Postbox","ua_name":"Postbox 1.1.3","ua_version":"1.1.3","ua_url":"http://www.postbox-inc.com/","ua_company":"Postbox, Inc.","ua_company_url":"http://www.postbox-inc.com/","ua_icon":"http://cdn.mandrill.com/img/email-client-icons/postbox.png","os_family":"OS X","os_name":"OS X 10.6 Snow Leopard","os_url":"http://www.apple.com/osx/","os_company":"Apple Computer, Inc.","os_company_url":"http://www.apple.com/","os_icon":"http://cdn.mandrill.com/img/email-client-icons/macosx.png","mobile":false},"url":"http://mandrill.com","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_clicked/jsonschema/1-0-1","data":{"ip":"127.0.0.1","location":{"city":"Oklahoma City","latitude":35.4675598145,"timezone":"-05:00","country":"United States","longitude":-97.5164337158,"country_short":"US","postal_code":"73101","region":"Oklahoma"},"url":"http://mandrill.com","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa5","ts":"2014-11-06T09:49:26.000Z","user_agent":"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.8) Gecko/20100317 Postbox/1.1.3","msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"sent","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa5","tags":["webhook-example"],"ts":"2013-04-04T21:13:19.000Z","clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","opens":[{"ts":"2013-04-04T21:31:51.000Z"}]},"user_agent_parsed":{"os_company_url":"http://www.apple.com/","os_family":"OS X","os_company":"Apple Computer, Inc.","os_url":"http://www.apple.com/osx/","ua_url":"http://www.postbox-inc.com/","ua_icon":"http://cdn.mandrill.com/img/email-client-icons/postbox.png","ua_version":"1.1.3","os_name":"OS X 10.6 Snow Leopard","ua_company":"Postbox, Inc.","ua_family":"Postbox","os_icon":"http://cdn.mandrill.com/img/email-client-icons/macosx.png","ua_company_url":"http://www.postbox-inc.com/","ua_name":"Postbox 1.1.3","type":"Email Client","mobile":false}}}}""" ), ContentType.some, Shared.cljSource, @@ -154,10 +164,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_marked_as_spam/jsonschema/1-0-1","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"opens":[{"ts":"2013-04-04T21:31:51.000Z"}],"clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"state":"sent","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa6","_version":"exampleaaaaaaaaaaaaaaa"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa6","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_marked_as_spam/jsonschema/1-0-1","data":{"msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"sent","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa6","tags":["webhook-example"],"ts":"2013-04-04T21:13:19.000Z","clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","opens":[{"ts":"2013-04-04T21:31:51.000Z"}]},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa6","ts":"2014-11-06T09:49:26.000Z"}}}""" ), ContentType.some, Shared.cljSource, @@ -166,10 +176,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/recipient_unsubscribed/jsonschema/1-0-1","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"opens":[{"ts":"2013-04-04T21:31:51.000Z"}],"clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"state":"sent","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa7","_version":"exampleaaaaaaaaaaaaaaa"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa7","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/recipient_unsubscribed/jsonschema/1-0-1","data":{"msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"sent","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa7","tags":["webhook-example"],"ts":"2013-04-04T21:13:19.000Z","clicks":[{"ts":"2013-04-04T21:31:51.000Z","url":"http://mandrill.com"}],"metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","opens":[{"ts":"2013-04-04T21:31:51.000Z"}]},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa7","ts":"2014-11-06T09:49:26.000Z"}}}""" ), ContentType.some, Shared.cljSource, @@ -178,10 +188,10 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM RawEvent( Shared.api, Map( - "tv" -> "com.mandrill-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_rejected/jsonschema/1-0-0","data":{"msg":{"ts":"2013-04-04T21:13:19.000Z","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","sender":"example.sender@mandrillapp.com","tags":["webhook-example"],"opens":[],"clicks":[],"state":"rejected","metadata":{"user_id":111},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa8","_version":"exampleaaaaaaaaaaaaaaa"},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa8","ts":"2014-11-06T09:49:26.000Z"}}}""" + "tv" -> "com.mandrill-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mandrill/message_rejected/jsonschema/1-0-0","data":{"msg":{"_version":"exampleaaaaaaaaaaaaaaa","subject":"This an example webhook message","email":"example.webhook@mandrillapp.com","state":"rejected","_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa8","tags":["webhook-example"],"ts":"2013-04-04T21:13:19.000Z","clicks":[],"metadata":{"user_id":111},"sender":"example.sender@mandrillapp.com","opens":[]},"_id":"exampleaaaaaaaaaaaaaaaaaaaaaaaaa8","ts":"2014-11-06T09:49:26.000Z"}}}""" ), ContentType.some, Shared.cljSource, @@ -194,7 +204,13 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM def e5 = { // Spec for nine seperate events where two have incorrect event names and one does not have event as a parameter val bodyStr = "mandrill_events=%5B%7B%22event%22%3A%22sending%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%5D%2C%22clicks%22%3A%5B%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22deferred%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%5D%2C%22clicks%22%3A%5B%5D%2C%22state%22%3A%22deferred%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa1%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%2C%22smtp_events%22%3A%5B%7B%22destination_ip%22%3A%22127.0.0.1%22%2C%22diag%22%3A%22451+4.3.5+Temporarily+unavailable%2C+try+again+later.%22%2C%22source_ip%22%3A%22127.0.0.1%22%2C%22ts%22%3A1365111111%2C%22type%22%3A%22deferred%22%2C%22size%22%3A0%7D%5D%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa1%22%2C%22ts%22%3A1415267366%7D%2C%7B%22eventsss%22%3A%22hard_bounce%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22state%22%3A%22bounced%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa2%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%2C%22bounce_description%22%3A%22bad_mailbox%22%2C%22bgtools_code%22%3A10%2C%22diag%22%3A%22smtp%3B550+5.1.1+The+email+account+that+you+tried+to+reach+does+not+exist.+Please+try+double-checking+the+recipient%27s+email+address+for+typos+or+unnecessary+spaces.%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa2%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22soft_bounce%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22state%22%3A%22soft-bounced%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa3%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%2C%22bounce_description%22%3A%22mailbox_full%22%2C%22bgtools_code%22%3A22%2C%22diag%22%3A%22smtp%3B552+5.2.2+Over+Quota%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa3%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22open%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa4%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa4%22%2C%22ip%22%3A%22127.0.0.1%22%2C%22location%22%3A%7B%22country_short%22%3A%22US%22%2C%22country%22%3A%22United+States%22%2C%22region%22%3A%22Oklahoma%22%2C%22city%22%3A%22Oklahoma+City%22%2C%22latitude%22%3A35.4675598145%2C%22longitude%22%3A-97.5164337158%2C%22postal_code%22%3A%2273101%22%2C%22timezone%22%3A%22-05%3A00%22%7D%2C%22user_agent%22%3A%22Mozilla%5C%2F5.0+%28Macintosh%3B+U%3B+Intel+Mac+OS+X+10.6%3B+en-US%3B+rv%3A1.9.1.8%29+Gecko%5C%2F20100317+Postbox%5C%2F1.1.3%22%2C%22user_agent_parsed%22%3A%7B%22type%22%3A%22Email+Client%22%2C%22ua_family%22%3A%22Postbox%22%2C%22ua_name%22%3A%22Postbox+1.1.3%22%2C%22ua_version%22%3A%221.1.3%22%2C%22ua_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_company%22%3A%22Postbox%2C+Inc.%22%2C%22ua_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fpostbox.png%22%2C%22os_family%22%3A%22OS+X%22%2C%22os_name%22%3A%22OS+X+10.6+Snow+Leopard%22%2C%22os_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2Fosx%5C%2F%22%2C%22os_company%22%3A%22Apple+Computer%2C+Inc.%22%2C%22os_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2F%22%2C%22os_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fmacosx.png%22%2C%22mobile%22%3Afalse%7D%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22click%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa5%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa5%22%2C%22ip%22%3A%22127.0.0.1%22%2C%22location%22%3A%7B%22country_short%22%3A%22US%22%2C%22country%22%3A%22United+States%22%2C%22region%22%3A%22Oklahoma%22%2C%22city%22%3A%22Oklahoma+City%22%2C%22latitude%22%3A35.4675598145%2C%22longitude%22%3A-97.5164337158%2C%22postal_code%22%3A%2273101%22%2C%22timezone%22%3A%22-05%3A00%22%7D%2C%22user_agent%22%3A%22Mozilla%5C%2F5.0+%28Macintosh%3B+U%3B+Intel+Mac+OS+X+10.6%3B+en-US%3B+rv%3A1.9.1.8%29+Gecko%5C%2F20100317+Postbox%5C%2F1.1.3%22%2C%22user_agent_parsed%22%3A%7B%22type%22%3A%22Email+Client%22%2C%22ua_family%22%3A%22Postbox%22%2C%22ua_name%22%3A%22Postbox+1.1.3%22%2C%22ua_version%22%3A%221.1.3%22%2C%22ua_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_company%22%3A%22Postbox%2C+Inc.%22%2C%22ua_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.postbox-inc.com%5C%2F%22%2C%22ua_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fpostbox.png%22%2C%22os_family%22%3A%22OS+X%22%2C%22os_name%22%3A%22OS+X+10.6+Snow+Leopard%22%2C%22os_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2Fosx%5C%2F%22%2C%22os_company%22%3A%22Apple+Computer%2C+Inc.%22%2C%22os_company_url%22%3A%22http%3A%5C%2F%5C%2Fwww.apple.com%5C%2F%22%2C%22os_icon%22%3A%22http%3A%5C%2F%5C%2Fcdn.mandrill.com%5C%2Fimg%5C%2Femail-client-icons%5C%2Fmacosx.png%22%2C%22mobile%22%3Afalse%7D%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22spam%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa6%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa6%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22unsub%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%7B%22ts%22%3A1365111111%7D%5D%2C%22clicks%22%3A%5B%7B%22ts%22%3A1365111111%2C%22url%22%3A%22http%3A%5C%2F%5C%2Fmandrill.com%22%7D%5D%2C%22state%22%3A%22sent%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa7%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa7%22%2C%22ts%22%3A1415267366%7D%2C%7B%22event%22%3A%22reject%22%2C%22msg%22%3A%7B%22ts%22%3A1365109999%2C%22subject%22%3A%22This+an+example+webhook+message%22%2C%22email%22%3A%22example.webhook%40mandrillapp.com%22%2C%22sender%22%3A%22example.sender%40mandrillapp.com%22%2C%22tags%22%3A%5B%22webhook-example%22%5D%2C%22opens%22%3A%5B%5D%2C%22clicks%22%3A%5B%5D%2C%22state%22%3A%22rejected%22%2C%22metadata%22%3A%7B%22user_id%22%3A111%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa8%22%2C%22_version%22%3A%22exampleaaaaaaaaaaaaaaa%22%7D%2C%22_id%22%3A%22exampleaaaaaaaaaaaaaaaaaaaaaaaaa8%22%2C%22ts%22%3A1415267366%7D%5D" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( "Mandrill event at index [0] failed: type parameter [sending] not recognized", "Mandrill event at index [1] failed: type parameter [deferred] not recognized", @@ -204,23 +220,25 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidationM } def e6 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) MandrillAdapter.toRawEvents(payload) must beFailing( NonEmptyList("Request body is empty: no Mandrill events to process")) } def e7 = { - val body = "mandrill_events=%5B%7B%22event%22%3A%20%22subscribe%22%7D%5D" - val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) - MandrillAdapter.toRawEvents(payload) must beFailing( - NonEmptyList( - "Request body provided but content type empty, expected application/x-www-form-urlencoded for Mandrill")) + val body = "mandrill_events=%5B%7B%22event%22%3A%20%22subscribe%22%7D%5D" + val payload = + CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) + MandrillAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Request body provided but content type empty, expected application/x-www-form-urlencoded for Mandrill")) } def e8 = { - val body = "mandrill_events=%5B%7B%22event%22%3A%20%22subscribe%22%7D%5D" - val ct = "application/x-www-form-urlencoded; charset=utf-8" - val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) + val body = "mandrill_events=%5B%7B%22event%22%3A%20%22subscribe%22%7D%5D" + val ct = "application/x-www-form-urlencoded; charset=utf-8" + val payload = + CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) MandrillAdapter.toRawEvents(payload) must beFailing(NonEmptyList( "Content type of application/x-www-form-urlencoded; charset=utf-8 provided, expected application/x-www-form-urlencoded for Mandrill")) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala index 1fda7fa3f..c6628e656 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala @@ -23,7 +23,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} -class MarketoAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class MarketoAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the MarketoAdapter functionality toRawEvents must return a success for a valid "event" type payload body being passed $e1 @@ -33,30 +37,37 @@ class MarketoAdapterSpec extends Specification with DataTables with ValidationMa implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.marketo", "v1") + val api = CollectorApi("com.marketo", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2018-01-01T00:00:00.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2018-01-01T00:00:00.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/json" def e1 = { val bodyStr = - """{"name": "webhook for A", "step": 6, "campaign": {"id": 160, "name": "avengers assemble"}, "lead": {"acquisition_date": "2010-11-11 11:11:11", "black_listed": false, "first_name": "the hulk", "updated_at": "", "created_at": "2018-06-16 11:23:58", "last_interesting_moment_date": "2018-09-26 20:26:40"}, "company": {"name": "iron man", "notes": "the something dog leapt over the lazy fox"}, "campaign": {"id": 987, "name": "triggered event"}, "datetime": "2018-03-07 14:28:16"}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + """{"name": "webhook for A", "step": 6, "lead": {"acquisition_date": "2010-11-11 11:11:11", "black_listed": false, "first_name": "the hulk", "updated_at": "", "created_at": "2018-06-16 11:23:58", "last_interesting_moment_date": "2018-09-26 20:26:40"}, "company": {"name": "iron man", "notes": "the something dog leapt over the lazy fox"}, "campaign": {"id": 987, "name": "triggered event"}, "datetime": "2018-03-07 14:28:16"}""" + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.marketo-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.marketo/event/jsonschema/2-0-0","data":{"name":"webhook for A","step":6,"campaign":{"id":160,"name":"avengers assemble"},"lead":{"acquisition_date":"2010-11-11T11:11:11.000Z","black_listed":false,"first_name":"the hulk","updated_at":"","created_at":"2018-06-16T11:23:58.000Z","last_interesting_moment_date":"2018-09-26T20:26:40.000Z"},"company":{"name":"iron man","notes":"the something dog leapt over the lazy fox"},"campaign":{"id":987,"name":"triggered event"},"datetime":"2018-03-07T14:28:16.000Z"}}}""" + "tv" -> "com.marketo-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.marketo/event/jsonschema/2-0-0","data":{"lead":{"first_name":"the hulk","acquisition_date":"2010-11-11T11:11:11.000Z","black_listed":false,"last_interesting_moment_date":"2018-09-26T20:26:40.000Z","created_at":"2018-06-16T11:23:58.000Z","updated_at":""},"name":"webhook for A","step":6,"campaign":{"id":987,"name":"triggered event"},"datetime":"2018-03-07T14:28:16.000Z","company":{"name":"iron man","notes":"the something dog leapt over the lazy fox"}}}}""" ), ContentType.some, Shared.cljSource, @@ -66,7 +77,8 @@ class MarketoAdapterSpec extends Specification with DataTables with ValidationMa } def e2 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) MarketoAdapter.toRawEvents(payload) must beFailing( NonEmptyList("Request body is empty: no Marketo event to process")) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala index 1b7471ebd..faf5ee85b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala @@ -23,7 +23,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} -class OlarkAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class OlarkAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the OlarkAdapter functionality toRawEvents must return a Success Nel if the transcript event in the payload is successful $e1 @@ -39,14 +43,15 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidationMatc implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.olark", "v1") + val api = CollectorApi("com.olark", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/x-www-form-urlencoded" @@ -54,7 +59,13 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidationMatc def e1 = { val body = "data=%7B%22kind%22%3A%20%22Conversation%22%2C%20%22tags%22%3A%20%5B%22olark%22%2C%20%22customer%22%5D%2C%20%22items%22%3A%20%5B%7B%22body%22%3A%20%22Hi%20there.%20Need%20any%20help%3F%22%2C%20%22timestamp%22%3A%20%221307116657.1%22%2C%20%22kind%22%3A%20%22MessageToVisitor%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22operatorId%22%3A%20%221234%22%7D%2C%20%7B%22body%22%3A%20%22Yes%2C%20please%20help%20me%20with%20billing.%22%2C%20%22timestamp%22%3A%20%221307116661.25%22%2C%20%22kind%22%3A%20%22MessageToOperator%22%2C%20%22nickname%22%3A%20%22Bob%22%7D%5D%2C%20%22operators%22%3A%20%7B%221234%22%3A%20%7B%22username%22%3A%20%22jdoe%22%2C%20%22emailAddress%22%3A%20%22john%40example.com%22%2C%20%22kind%22%3A%20%22Operator%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22id%22%3A%20%221234%22%7D%7D%2C%20%22groups%22%3A%20%5B%7B%22kind%22%3A%20%22Group%22%2C%20%22name%22%3A%20%22My%20Sales%20Group%22%2C%20%22id%22%3A%20%220123456789abcdef%22%7D%5D%2C%20%22visitor%22%3A%20%7B%22ip%22%3A%20%22123.4.56.78%22%2C%20%22city%22%3A%20%22Palo%20Alto%22%2C%20%22kind%22%3A%20%22Visitor%22%2C%20%22conversationBeginPage%22%3A%20%22http%3A%2F%2Fwww.example.com%2Fpath%22%2C%20%22countryCode%22%3A%20%22US%22%2C%20%22country%22%3A%20%22United%20State%22%2C%20%22region%22%3A%20%22CA%22%2C%20%22chat_feedback%22%3A%20%7B%22overall_chat%22%3A%205%2C%20%22responsiveness%22%3A%205%2C%20%22friendliness%22%3A%205%2C%20%22knowledge%22%3A%205%2C%20%22comments%22%3A%20%22Very%20helpful%2C%20thanks%22%7D%2C%20%22operatingSystem%22%3A%20%22Windows%22%2C%20%22emailAddress%22%3A%20%22bob%40example.com%22%2C%20%22organization%22%3A%20%22Widgets%20Inc.%22%2C%20%22phoneNumber%22%3A%20%22%28555%29%20555-5555%22%2C%20%22fullName%22%3A%20%22Bob%20Doe%22%2C%20%22customFields%22%3A%20%7B%22favoriteColor%22%3A%20%22blue%22%2C%20%22myInternalCustomerId%22%3A%20%2212341234%22%7D%2C%20%22id%22%3A%20%229QRF9YWM5XW3ZSU7P9CGWRU89944341%22%2C%20%22browser%22%3A%20%22Chrome%2012.1%22%7D%2C%20%22id%22%3A%20%22EV695BI2930A6XMO32886MPT899443414%22%7D" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -121,18 +132,25 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidationMatc |} |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.olark-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.olark-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) OlarkAdapter.toRawEvents(payload) must beSuccessful(expected) } def e2 = { val body = "data=%7B%22kind%22%3A%20%22Conversation%22%2C%20%22id%22%3A%20%22EV695BI2930A6XMO32886QPT899443414%22%2C%20%22items%22%3A%20%5B%7B%22kind%22%3A%20%22OfflineMessage%22%2C%20%22timestamp%22%3A%20%221307116667.1%22%2C%20%22body%22%3A%20%22Hi%20there.%22%7D%5D%2C%20%22visitor%22%3A%20%7B%22kind%22%3A%20%22Visitor%22%2C%20%22id%22%3A%20%229QRF9YWM5XW3ZSU7P9CGWRU89944341%22%2C%20%22fullName%22%3A%20%22John%20Doe%22%2C%20%22emailAddress%22%3A%20%22foo%40example.com%22%2C%20%22phoneNumber%22%3A%20%22%28555%29%20555-5555%22%2C%20%22city%22%3A%20%22Palo%20Alto%22%2C%20%22region%22%3A%20%22CA%22%2C%20%22country%22%3A%20%22United%20States%22%2C%20%22countryCode%22%3A%20%22US%22%2C%20%22organization%22%3A%20%22Widgets%20Inc.%22%2C%20%22ip%22%3A%20%22123.4.56.78%22%2C%20%22browser%22%3A%20%22Chrome%2012.1%22%2C%20%22operatingSystem%22%3A%20%22Windows%22%2C%20%22customFields%22%3A%20%7B%22myInternalCustomerId%22%3A%20%2212341234%22%2C%20%22favoriteColor%22%3A%20%22blue%22%7D%20%7D%2C%20%22groups%22%3A%20%5B%7B%22name%22%3A%20%22My%20Sales%20Group%22%2C%20%22id%22%3A%20%220123456789abcdef%22%2C%20%22kind%22%3A%20%22Group%22%7D%5D%20%7D" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -175,40 +193,50 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidationMatc |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.olark-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.olark-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) OlarkAdapter.toRawEvents(payload) must beSuccessful(expected) } def e3 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) - OlarkAdapter.toRawEvents(payload) must beFailing(NonEmptyList("Request body is empty: no Olark events to process")) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + OlarkAdapter.toRawEvents(payload) must beFailing( + NonEmptyList("Request body is empty: no Olark events to process")) } def e4 = { val body = "data=%7B%22kind%22%3A%20%22Conversation%22%2C%20%22tags%22%3A%20%5B%22olark%22%2C%20%22customer%22%5D%2C%20%22items%22%3A%20%5B%7B%22body%22%3A%20%22Hi%20there.%20Need%20any%20help%3F%22%2C%20%22timestamp%22%3A%20%221307116657.1%22%2C%20%22kind%22%3A%20%22MessageToVisitor%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22operatorId%22%3A%20%221234%22%7D%2C%20%7B%22body%22%3A%20%22Yes%2C%20please%20help%20me%20with%20billing.%22%2C%20%22timestamp%22%3A%20%221307116661.25%22%2C%20%22kind%22%3A%20%22MessageToOperator%22%2C%20%22nickname%22%3A%20%22Bob%22%7D%5D%2C%20%22operators%22%3A%20%7B%221234%22%3A%20%7B%22username%22%3A%20%22jdoe%22%2C%20%22emailAddress%22%3A%20%22john%40example.com%22%2C%20%22kind%22%3A%20%22Operator%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22id%22%3A%20%221234%22%7D%7D%2C%20%22groups%22%3A%20%5B%7B%22kind%22%3A%20%22Group%22%2C%20%22name%22%3A%20%22My%20Sales%20Group%22%2C%20%22id%22%3A%20%220123456789abcdef%22%7D%5D%2C%20%22visitor%22%3A%20%7B%22ip%22%3A%20%22123.4.56.78%22%2C%20%22city%22%3A%20%22Palo%20Alto%22%2C%20%22kind%22%3A%20%22Visitor%22%2C%20%22conversationBeginPage%22%3A%20%22http%3A%2F%2Fwww.example.com%2Fpath%22%2C%20%22countryCode%22%3A%20%22US%22%2C%20%22country%22%3A%20%22United%20State%22%2C%20%22region%22%3A%20%22CA%22%2C%20%22chat_feedback%22%3A%20%7B%22overall_chat%22%3A%205%2C%20%22responsiveness%22%3A%205%2C%20%22friendliness%22%3A%205%2C%20%22knowledge%22%3A%205%2C%20%22comments%22%3A%20%22Very%20helpful%2C%20thanks%22%7D%2C%20%22operatingSystem%22%3A%20%22Windows%22%2C%20%22emailAddress%22%3A%20%22bob%40example.com%22%2C%20%22organization%22%3A%20%22Widgets%20Inc.%22%2C%20%22phoneNumber%22%3A%20%22%28555%29%20555-5555%22%2C%20%22fullName%22%3A%20%22Bob%20Doe%22%2C%20%22customFields%22%3A%20%7B%22favoriteColor%22%3A%20%22blue%22%2C%20%22myInternalCustomerId%22%3A%20%2212341234%22%7D%2C%20%22id%22%3A%20%229QRF9YWM5XW3ZSU7P9CGWRU89944341%22%2C%20%22browser%22%3A%20%22Chrome%2012.1%22%7D%2C%20%22id%22%3A%20%22EV695BI2930A6XMO32886MPT899443414%22%7D" - val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) - OlarkAdapter.toRawEvents(payload) must beFailing( - NonEmptyList( - "Request body provided but content type empty, expected application/x-www-form-urlencoded for Olark")) + val payload = + CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) + OlarkAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Request body provided but content type empty, expected application/x-www-form-urlencoded for Olark")) } def e5 = { val body = "data=%7B%22kind%22%3A%20%22Conversation%22%2C%20%22tags%22%3A%20%5B%22olark%22%2C%20%22customer%22%5D%2C%20%22items%22%3A%20%5B%7B%22body%22%3A%20%22Hi%20there.%20Need%20any%20help%3F%22%2C%20%22timestamp%22%3A%20%221307116657.1%22%2C%20%22kind%22%3A%20%22MessageToVisitor%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22operatorId%22%3A%20%221234%22%7D%2C%20%7B%22body%22%3A%20%22Yes%2C%20please%20help%20me%20with%20billing.%22%2C%20%22timestamp%22%3A%20%221307116661.25%22%2C%20%22kind%22%3A%20%22MessageToOperator%22%2C%20%22nickname%22%3A%20%22Bob%22%7D%5D%2C%20%22operators%22%3A%20%7B%221234%22%3A%20%7B%22username%22%3A%20%22jdoe%22%2C%20%22emailAddress%22%3A%20%22john%40example.com%22%2C%20%22kind%22%3A%20%22Operator%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22id%22%3A%20%221234%22%7D%7D%2C%20%22groups%22%3A%20%5B%7B%22kind%22%3A%20%22Group%22%2C%20%22name%22%3A%20%22My%20Sales%20Group%22%2C%20%22id%22%3A%20%220123456789abcdef%22%7D%5D%2C%20%22visitor%22%3A%20%7B%22ip%22%3A%20%22123.4.56.78%22%2C%20%22city%22%3A%20%22Palo%20Alto%22%2C%20%22kind%22%3A%20%22Visitor%22%2C%20%22conversationBeginPage%22%3A%20%22http%3A%2F%2Fwww.example.com%2Fpath%22%2C%20%22countryCode%22%3A%20%22US%22%2C%20%22country%22%3A%20%22United%20State%22%2C%20%22region%22%3A%20%22CA%22%2C%20%22chat_feedback%22%3A%20%7B%22overall_chat%22%3A%205%2C%20%22responsiveness%22%3A%205%2C%20%22friendliness%22%3A%205%2C%20%22knowledge%22%3A%205%2C%20%22comments%22%3A%20%22Very%20helpful%2C%20thanks%22%7D%2C%20%22operatingSystem%22%3A%20%22Windows%22%2C%20%22emailAddress%22%3A%20%22bob%40example.com%22%2C%20%22organization%22%3A%20%22Widgets%20Inc.%22%2C%20%22phoneNumber%22%3A%20%22%28555%29%20555-5555%22%2C%20%22fullName%22%3A%20%22Bob%20Doe%22%2C%20%22customFields%22%3A%20%7B%22favoriteColor%22%3A%20%22blue%22%2C%20%22myInternalCustomerId%22%3A%20%2212341234%22%7D%2C%20%22id%22%3A%20%229QRF9YWM5XW3ZSU7P9CGWRU89944341%22%2C%20%22browser%22%3A%20%22Chrome%2012.1%22%7D%2C%20%22id%22%3A%20%22EV695BI2930A6XMO32886MPT899443414%22%7D" - val ct = "application/json" - val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) - OlarkAdapter.toRawEvents(payload) must beFailing( - NonEmptyList("Content type of application/json provided, expected application/x-www-form-urlencoded for Olark")) + val ct = "application/json" + val payload = + CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) + OlarkAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Content type of application/json provided, expected application/x-www-form-urlencoded for Olark")) } def e6 = { - val body = "" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val body = "" + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Olark event body is empty: nothing to process") OlarkAdapter.toRawEvents(payload) must beFailing(expected) } @@ -216,7 +244,13 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidationMatc def e7 = { val body = "%7B%22kind%22%3A%20%22Conversation%22%2C%20%22tags%22%3A%20%5B%22olark%22%2C%20%22customer%22%5D%2C%20%22items%22%3A%20%5B%7B%22body%22%3A%20%22Hi%20there.%20Need%20any%20help%3F%22%2C%20%22timestamp%22%3A%20%221307116657.1%22%2C%20%22kind%22%3A%20%22MessageToVisitor%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22operatorId%22%3A%20%221234%22%7D%2C%20%7B%22body%22%3A%20%22Yes%2C%20please%20help%20me%20with%20billing.%22%2C%20%22timestamp%22%3A%20%221307116661.25%22%2C%20%22kind%22%3A%20%22MessageToOperator%22%2C%20%22nickname%22%3A%20%22Bob%22%7D%5D%2C%20%22operators%22%3A%20%7B%221234%22%3A%20%7B%22username%22%3A%20%22jdoe%22%2C%20%22emailAddress%22%3A%20%22john%40example.com%22%2C%20%22kind%22%3A%20%22Operator%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22id%22%3A%20%221234%22%7D%7D%2C%20%22groups%22%3A%20%5B%7B%22kind%22%3A%20%22Group%22%2C%20%22name%22%3A%20%22My%20Sales%20Group%22%2C%20%22id%22%3A%20%220123456789abcdef%22%7D%5D%2C%20%22visitor%22%3A%20%7B%22ip%22%3A%20%22123.4.56.78%22%2C%20%22city%22%3A%20%22Palo%20Alto%22%2C%20%22kind%22%3A%20%22Visitor%22%2C%20%22conversationBeginPage%22%3A%20%22http%3A%2F%2Fwww.example.com%2Fpath%22%2C%20%22countryCode%22%3A%20%22US%22%2C%20%22country%22%3A%20%22United%20State%22%2C%20%22region%22%3A%20%22CA%22%2C%20%22chat_feedback%22%3A%20%7B%22overall_chat%22%3A%205%2C%20%22responsiveness%22%3A%205%2C%20%22friendliness%22%3A%205%2C%20%22knowledge%22%3A%205%2C%20%22comments%22%3A%20%22Very%20helpful%2C%20thanks%22%7D%2C%20%22operatingSystem%22%3A%20%22Windows%22%2C%20%22emailAddress%22%3A%20%22bob%40example.com%22%2C%20%22organization%22%3A%20%22Widgets%20Inc.%22%2C%20%22phoneNumber%22%3A%20%22%28555%29%20555-5555%22%2C%20%22fullName%22%3A%20%22Bob%20Doe%22%2C%20%22customFields%22%3A%20%7B%22favoriteColor%22%3A%20%22blue%22%2C%20%22myInternalCustomerId%22%3A%20%2212341234%22%7D%2C%20%22id%22%3A%20%229QRF9YWM5XW3ZSU7P9CGWRU89944341%22%2C%20%22browser%22%3A%20%22Chrome%2012.1%22%7D%2C%20%22id%22%3A%20%22EV695BI2930A6XMO32886MPT899443414%22%7D" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Olark event data does not have 'data' as a key") OlarkAdapter.toRawEvents(payload) must beFailing(expected) } @@ -224,9 +258,15 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidationMatc def e8 = { val body = "data=kind%22%3A%20%22Conversation%22%2C%20%22tags%22%3A%20%5B%22olark%22%2C%20%22customer%22%5D%2C%20%22items%22%3A%20%5B%7B%22body%22%3A%20%22Hi%20there.%20Need%20any%20help%3F%22%2C%20%22timestamp%22%3A%20%221307116657.1%22%2C%20%22kind%22%3A%20%22MessageToVisitor%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22operatorId%22%3A%20%221234%22%7D%2C%20%7B%22body%22%3A%20%22Yes%2C%20please%20help%20me%20with%20billing.%22%2C%20%22timestamp%22%3A%20%221307116661.25%22%2C%20%22kind%22%3A%20%22MessageToOperator%22%2C%20%22nickname%22%3A%20%22Bob%22%7D%5D%2C%20%22operators%22%3A%20%7B%221234%22%3A%20%7B%22username%22%3A%20%22jdoe%22%2C%20%22emailAddress%22%3A%20%22john%40example.com%22%2C%20%22kind%22%3A%20%22Operator%22%2C%20%22nickname%22%3A%20%22John%22%2C%20%22id%22%3A%20%221234%22%7D%7D%2C%20%22groups%22%3A%20%5B%7B%22kind%22%3A%20%22Group%22%2C%20%22name%22%3A%20%22My%20Sales%20Group%22%2C%20%22id%22%3A%20%220123456789abcdef%22%7D%5D%2C%20%22visitor%22%3A%20%7B%22ip%22%3A%20%22123.4.56.78%22%2C%20%22city%22%3A%20%22Palo%20Alto%22%2C%20%22kind%22%3A%20%22Visitor%22%2C%20%22conversationBeginPage%22%3A%20%22http%3A%2F%2Fwww.example.com%2Fpath%22%2C%20%22countryCode%22%3A%20%22US%22%2C%20%22country%22%3A%20%22United%20State%22%2C%20%22region%22%3A%20%22CA%22%2C%20%22chat_feedback%22%3A%20%7B%22overall_chat%22%3A%205%2C%20%22responsiveness%22%3A%205%2C%20%22friendliness%22%3A%205%2C%20%22knowledge%22%3A%205%2C%20%22comments%22%3A%20%22Very%20helpful%2C%20thanks%22%7D%2C%20%22operatingSystem%22%3A%20%22Windows%22%2C%20%22emailAddress%22%3A%20%22bob%40example.com%22%2C%20%22organization%22%3A%20%22Widgets%20Inc.%22%2C%20%22phoneNumber%22%3A%20%22%28555%29%20555-5555%22%2C%20%22fullName%22%3A%20%22Bob%20Doe%22%2C%20%22customFields%22%3A%20%7B%22favoriteColor%22%3A%20%22blue%22%2C%20%22myInternalCustomerId%22%3A%20%2212341234%22%7D%2C%20%22id%22%3A%20%229QRF9YWM5XW3ZSU7P9CGWRU89944341%22%2C%20%22browser%22%3A%20%22Chrome%2012.1%22%7D%2C%20%22id%22%3A%20%22EV695BI2930A6XMO32886MPT899443414%22%7D" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( - """Olark event string failed to parse into JSON: [Unrecognized token 'kind': was expecting ('true', 'false' or 'null') at [Source: (String)"kind": "Conversation", "tags": ["olark", "customer"], "items": [{"body": "Hi there. Need any help?", "timestamp": "1307116657.1", "kind": "MessageToVisitor", "nickname": "John", "operatorId": "1234"}, {"body": "Yes, please help me with billing.", "timestamp": "1307116661.25", "kind": "MessageToOperator", "nickname": "Bob"}], "operators": {"1234": {"username": "jdoe", "emailAddress": "john@example.com", "kind": "Operator", "nickname": "John", "id": "1234"}}, "groups": [{"kind": "Group", "name": ""[truncated 710 chars]; line: 1, column: 5]]""") + """Olark event string failed to parse into JSON: [expected json value got 'kind":...' (line 1, column 1)]""") OlarkAdapter.toRawEvents(payload) must beFailing(expected) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala index 2e906f2bb..69c4b3391 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala @@ -14,18 +14,22 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry +import io.circe.literal._ import org.joda.time.DateTime -import org.specs2.{Specification, ScalaCheck} +import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders._ -class PagerdutyAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { def is = s2""" +class PagerdutyAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { + def is = s2""" This is a specification to test the PagerdutyAdapter functionality reformatParameters must return an updated JSON whereby all null Strings have been replaced by null $e1 reformatParameters must return an updated JSON where 'incident.xxx' is replaced by xxx $e2 @@ -44,72 +48,127 @@ class PagerdutyAdapterSpec extends Specification with DataTables with Validation object Shared { val api = CollectorApi("com.pagerduty", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, "37.157.33.123".some, None, None, Nil, None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/json" def e1 = - "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | - "Valid, update one value" !! """{"type":"null"}""" ! """{"type":null}""" | - "Valid, update multiple values" !! """{"type":"null","some":"null"}""" ! """{"type":null,"some":null}""" | - "Valid, update nested values" !! """{"type": {"some":"null"}}""" ! """{"type":{"some":null}}""" |> { - (_, input, expected) => PagerdutyAdapter.reformatParameters(parse(input)) mustEqual parse(expected) - } + "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | + "Valid, update one value" !! json"""{"type":"null"}""" ! json"""{"type":null}""" | + "Valid, update multiple values" !! json"""{"type":"null","some":"null"}""" ! json"""{"type":null,"some":null}""" | + "Valid, update nested values" !! json"""{"type": {"some":"null"}}""" ! json"""{"type":{"some":null}}""" |> { + (_, input, expected) => + PagerdutyAdapter.reformatParameters(input) mustEqual expected + } def e2 = { - val json = parse("""{"type":"incident.trigger"}""") - val expected = parse("""{"type":"trigger"}""") + val json = json"""{"type":"incident.trigger"}""" + val expected = json"""{"type":"trigger"}""" PagerdutyAdapter.reformatParameters(json) mustEqual expected } def e3 = - "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | - "Valid, update one value" !! """{"created_on":"2014-11-12T18:53:47 00:00"}""" ! """{"created_on":"2014-11-12T18:53:47+00:00"}""" | - "Valid, update multiple values" !! """{"created_on":"2014-11-12T18:53:47 00:00","last_status_change_on":"2014-11-12T18:53:47 00:00"}""" ! """{"created_on":"2014-11-12T18:53:47+00:00","last_status_change_on":"2014-11-12T18:53:47+00:00"}""" | - "Valid, update nested values" !! """{"created_on":"2014-12-15T08:19:54Z","nested":{"created_on":"2014-11-12T18:53:47 00:00"}}""" ! """{"created_on":"2014-12-15T08:19:54Z","nested":{"created_on":"2014-11-12T18:53:47+00:00"}}""" |> { - (_, input, expected) => PagerdutyAdapter.reformatParameters(parse(input)) mustEqual parse(expected) - } + "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | + "Valid, update one value" !! json"""{"created_on":"2014-11-12T18:53:47 00:00"}""" ! json"""{"created_on":"2014-11-12T18:53:47+00:00"}""" | + "Valid, update multiple values" !! json"""{"created_on":"2014-11-12T18:53:47 00:00","last_status_change_on":"2014-11-12T18:53:47 00:00"}""" ! json"""{"created_on":"2014-11-12T18:53:47+00:00","last_status_change_on":"2014-11-12T18:53:47+00:00"}""" | + "Valid, update nested values" !! json"""{"created_on":"2014-12-15T08:19:54Z","nested":{"created_on":"2014-11-12T18:53:47 00:00"}}""" ! json"""{"created_on":"2014-12-15T08:19:54Z","nested":{"created_on":"2014-11-12T18:53:47+00:00"}}""" |> { + (_, input, expected) => + PagerdutyAdapter.reformatParameters(input) mustEqual expected + } def e4 = { - val bodyStr = """{"messages":[{"type":"incident.trigger","data":{"incident":{"id":"P9WY9U9"}}}]}""" - val expected = List(JObject(List(("type",JString("incident.trigger")), ("data",JObject(List(("incident",JObject(List(("id",JString("P9WY9U9"))))))))))) + val bodyStr = + """{"messages":[{"type":"incident.trigger","data":{"incident":{"id":"P9WY9U9"}}}]}""" + val expected = List(json"""{ + "type": "incident.trigger", + "data": { + "incident": { + "id": "P9WY9U9" + } + } + }""") PagerdutyAdapter.payloadBodyToEvents(bodyStr) must beSuccessful(expected) } def e5 = - "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | - "Failure, parse exception" !! """{"something:"some"}""" ! """PagerDuty payload failed to parse into JSON: [com.fasterxml.jackson.core.JsonParseException: Unexpected character ('s' (code 115)): was expecting a colon to separate field name and value at [Source: (String)"{"something:"some"}"; line: 1, column: 15]]""" | - "Failure, missing messages key" !! """{"somekey":"key"}""" ! "PagerDuty payload does not contain the needed 'messages' key" |> { - (_, input, expected) => PagerdutyAdapter.payloadBodyToEvents(input) must beFailing(expected) + "SPEC NAME" || "INPUT" | "EXPECTED OUTPUT" | + "Failure, parse exception" !! """{"something:"some"}""" ! """PagerDuty payload failed to parse into JSON: [expected : got 'some"}' (line 1, column 14)]""" | + "Failure, missing messages key" !! """{"somekey":"key"}""" ! "Could not resolve PagerDuty payload into a JSON array of events" |> { + (_, input, expected) => + PagerdutyAdapter.payloadBodyToEvents(input) must beFailing(expected) } def e6 = { - val bodyStr = """{"messages":[{"type":"incident.trigger","data":{"incident":{"id":"P9WY9U9","incident_number":139,"created_on":"2014-11-12T18:53:47 00:00","status":"triggered","html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9","incident_key":"srv01/HTTP","service":{"id":"PTHO4FF","name":"Webhooks Test","html_url":"https://snowplow.pagerduty.com/services/PTHO4FF","deleted_at":null},"escalation_policy":{"id":"P8ETVHU","name":"Default","deleted_at":null},"assigned_to_user":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X"},"trigger_summary_data":{"description":"FAILURE for production/HTTP on machine srv01.acme.com","client":"Sample Monitoring Service","client_url":"https://monitoring.service.com"},"trigger_details_html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9/log_entries/P5AWPTR","trigger_type":"trigger_svc_event","last_status_change_on":"2014-11-12T18:53:47Z","last_status_change_by":null,"number_of_escalations":0,"assigned_to":[{"at":"2014-11-12T18:53:47Z","object":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X","type":"user"}}]}},"id":"3c3e8ee0-6a9d-11e4-b3d5-22000ae31361","created_on":"2014-11-12T18:53:47Z"}]}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) - val expected = NonEmptyList(RawEvent(Shared.api,Map("tv" -> "com.pagerduty-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.pagerduty/incident/jsonschema/1-0-0","data":{"type":"trigger","data":{"incident":{"id":"P9WY9U9","incident_number":139,"created_on":"2014-11-12T18:53:47+00:00","status":"triggered","html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9","incident_key":"srv01/HTTP","service":{"id":"PTHO4FF","name":"Webhooks Test","html_url":"https://snowplow.pagerduty.com/services/PTHO4FF","deleted_at":null},"escalation_policy":{"id":"P8ETVHU","name":"Default","deleted_at":null},"assigned_to_user":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X"},"trigger_summary_data":{"description":"FAILURE for production/HTTP on machine srv01.acme.com","client":"Sample Monitoring Service","client_url":"https://monitoring.service.com"},"trigger_details_html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9/log_entries/P5AWPTR","trigger_type":"trigger_svc_event","last_status_change_on":"2014-11-12T18:53:47Z","last_status_change_by":null,"number_of_escalations":0,"assigned_to":[{"at":"2014-11-12T18:53:47Z","object":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X","type":"user"}}]}},"id":"3c3e8ee0-6a9d-11e4-b3d5-22000ae31361","created_on":"2014-11-12T18:53:47Z"}}}"""),ContentType.some, Shared.cljSource, Shared.context)) + val bodyStr = + """{"messages":[{"type":"incident.trigger","data":{"incident":{"id":"P9WY9U9","incident_number":139,"created_on":"2014-11-12T18:53:47 00:00","status":"triggered","html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9","incident_key":"srv01/HTTP","service":{"id":"PTHO4FF","name":"Webhooks Test","html_url":"https://snowplow.pagerduty.com/services/PTHO4FF","deleted_at":null},"escalation_policy":{"id":"P8ETVHU","name":"Default","deleted_at":null},"assigned_to_user":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X"},"trigger_summary_data":{"description":"FAILURE for production/HTTP on machine srv01.acme.com","client":"Sample Monitoring Service","client_url":"https://monitoring.service.com"},"trigger_details_html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9/log_entries/P5AWPTR","trigger_type":"trigger_svc_event","last_status_change_on":"2014-11-12T18:53:47Z","last_status_change_by":null,"number_of_escalations":0,"assigned_to":[{"at":"2014-11-12T18:53:47Z","object":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X","type":"user"}}]}},"id":"3c3e8ee0-6a9d-11e4-b3d5-22000ae31361","created_on":"2014-11-12T18:53:47Z"}]}""" + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) + val expected = NonEmptyList( + RawEvent( + Shared.api, + Map( + "tv" -> "com.pagerduty-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.pagerduty/incident/jsonschema/1-0-0","data":{"type":"trigger","data":{"incident":{"assigned_to_user":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X"},"incident_key":"srv01/HTTP","trigger_summary_data":{"description":"FAILURE for production/HTTP on machine srv01.acme.com","client":"Sample Monitoring Service","client_url":"https://monitoring.service.com"},"last_status_change_by":null,"incident_number":139,"service":{"id":"PTHO4FF","name":"Webhooks Test","html_url":"https://snowplow.pagerduty.com/services/PTHO4FF","deleted_at":null},"trigger_details_html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9/log_entries/P5AWPTR","id":"P9WY9U9","assigned_to":[{"at":"2014-11-12T18:53:47Z","object":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X","type":"user"}}],"number_of_escalations":0,"last_status_change_on":"2014-11-12T18:53:47Z","status":"triggered","escalation_policy":{"id":"P8ETVHU","name":"Default","deleted_at":null},"created_on":"2014-11-12T18:53:47+00:00","trigger_type":"trigger_svc_event","html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9"}},"id":"3c3e8ee0-6a9d-11e4-b3d5-22000ae31361","created_on":"2014-11-12T18:53:47Z"}}}""" + ), + ContentType.some, + Shared.cljSource, + Shared.context + )) PagerdutyAdapter.toRawEvents(payload) must beSuccessful(expected) } def e7 = { - val bodyStr = """{"messages":[{"type":"trigger","data":{"incident":{"id":"P9WY9U9","incident_number":139,"created_on":"2014-11-12T18:53:47 00:00","status":"triggered","html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9","incident_key":"srv01/HTTP","service":{"id":"PTHO4FF","name":"Webhooks Test","html_url":"https://snowplow.pagerduty.com/services/PTHO4FF","deleted_at":null},"escalation_policy":{"id":"P8ETVHU","name":"Default","deleted_at":null},"assigned_to_user":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X"},"trigger_summary_data":{"description":"FAILURE for production/HTTP on machine srv01.acme.com","client":"Sample Monitoring Service","client_url":"https://monitoring.service.com"},"trigger_details_html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9/log_entries/P5AWPTR","trigger_type":"trigger_svc_event","last_status_change_on":"2014-11-12T18:53:47Z","last_status_change_by":null,"number_of_escalations":0,"assigned_to":[{"at":"2014-11-12T18:53:47Z","object":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X","type":"user"}}]}},"id":"3c3e8ee0-6a9d-11e4-b3d5-22000ae31361","created_on":"2014-11-12T18:53:47Z"}]}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val bodyStr = + """{"messages":[{"type":"trigger","data":{"incident":{"id":"P9WY9U9","incident_number":139,"created_on":"2014-11-12T18:53:47 00:00","status":"triggered","html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9","incident_key":"srv01/HTTP","service":{"id":"PTHO4FF","name":"Webhooks Test","html_url":"https://snowplow.pagerduty.com/services/PTHO4FF","deleted_at":null},"escalation_policy":{"id":"P8ETVHU","name":"Default","deleted_at":null},"assigned_to_user":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X"},"trigger_summary_data":{"description":"FAILURE for production/HTTP on machine srv01.acme.com","client":"Sample Monitoring Service","client_url":"https://monitoring.service.com"},"trigger_details_html_url":"https://snowplow.pagerduty.com/incidents/P9WY9U9/log_entries/P5AWPTR","trigger_type":"trigger_svc_event","last_status_change_on":"2014-11-12T18:53:47Z","last_status_change_by":null,"number_of_escalations":0,"assigned_to":[{"at":"2014-11-12T18:53:47Z","object":{"id":"P9L426X","name":"Yali Sassoon","email":"yali@snowplowanalytics.com","html_url":"https://snowplow.pagerduty.com/users/P9L426X","type":"user"}}]}},"id":"3c3e8ee0-6a9d-11e4-b3d5-22000ae31361","created_on":"2014-11-12T18:53:47Z"}]}""" + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = "PagerDuty event at index [0] failed: type parameter [trigger] not recognized" PagerdutyAdapter.toRawEvents(payload) must beFailing(NonEmptyList(expected)) } def e8 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) - PagerdutyAdapter.toRawEvents(payload) must beFailing(NonEmptyList("Request body is empty: no PagerDuty events to process")) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + PagerdutyAdapter.toRawEvents(payload) must beFailing( + NonEmptyList("Request body is empty: no PagerDuty events to process")) } def e9 = { - val payload = CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) - PagerdutyAdapter.toRawEvents(payload) must beFailing(NonEmptyList("Request body provided but content type empty, expected application/json for PagerDuty")) + val payload = + CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) + PagerdutyAdapter.toRawEvents(payload) must beFailing( + NonEmptyList( + "Request body provided but content type empty, expected application/json for PagerDuty")) } def e10 = { - val payload = CollectorPayload(Shared.api, Nil, "application/x-www-form-urlencoded".some, "stub".some, Shared.cljSource, Shared.context) - PagerdutyAdapter.toRawEvents(payload) must beFailing(NonEmptyList("Content type of application/x-www-form-urlencoded provided, expected application/json for PagerDuty")) + val payload = CollectorPayload( + Shared.api, + Nil, + "application/x-www-form-urlencoded".some, + "stub".some, + Shared.cljSource, + Shared.context) + PagerdutyAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Content type of application/x-www-form-urlencoded provided, expected application/json for PagerDuty")) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala index 9d1a93ccb..ac1d9ea28 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala @@ -14,83 +14,73 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters package registry +import io.circe.literal._ import org.joda.time.DateTime import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} import SpecHelpers._ -class PingdomAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class PingdomAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the PingdomAdapter functionality reformatParameters should return either an updated JSON without the 'action' field or the same JSON $e1 - parseJson must return a Success Nel for a valid json string being passed $e2 - parseJson must return a Failure Nel containing the JsonParseException for invalid json strings $e3 - reformatMapParams must return a Failure Nel for any Python Unicode wrapped values $e4 - toRawEvents must return a Success Nel for a valid querystring $e5 - toRawEvents must return a Failure Nel for an empty querystring $e6 - toRawEvents must return a Failure Nel for a querystring which does not contain 'message' as a key $e7 + reformatMapParams must return a Failure Nel for any Python Unicode wrapped values $e2 + toRawEvents must return a Success Nel for a valid querystring $e3 + toRawEvents must return a Failure Nel for an empty querystring $e4 + toRawEvents must return a Failure Nel for a querystring which does not contain 'message' as a key $e5 """ implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.pingdom", "v1") + val api = CollectorApi("com.pingdom", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } def e1 = - "SPEC NAME" || "JSON" | "EXPECTED OUTPUT" | - "Remove action field" !! """{"action":"assign","agent":"smith"}""" ! """{"agent":"smith"}""" | - "Nothing removed" !! """{"actions":"assign","agent":"smith"}""" ! """{"actions":"assign","agent":"smith"}""" |> { + "SPEC NAME" || "JSON" | "EXPECTED OUTPUT" | + "Remove action field" !! json"""{"action":"assign","agent":"smith"}""" ! json"""{"agent":"smith"}""" | + "Nothing removed" !! json"""{"actions":"assign","agent":"smith"}""" ! json"""{"actions":"assign","agent":"smith"}""" |> { (_, json, expected) => - PingdomAdapter.reformatParameters(parse(json)) mustEqual parse(expected) + PingdomAdapter.reformatParameters(json) mustEqual expected } def e2 = { - val jsonStr = """{"event":"incident_assign"}""" - val expected = JObject(List(("event", JString("incident_assign")))) - PingdomAdapter.parseJson(jsonStr) must beSuccessful(expected) - } - - def e3 = { - val jsonStr = """{"event":incident_assign"}""" - val expected = - """Pingdom event failed to parse into JSON: [com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'incident_assign': was expecting ('true', 'false' or 'null') at [Source: (String)"{"event":incident_assign"}"; line: 1, column: 25]]""" - PingdomAdapter.parseJson(jsonStr) must beFailing(NonEmptyList(expected)) - } - - def e4 = { val nvPairs = toNameValuePairs("p" -> "(u'apps',)") val expected = "Pingdom name-value pair [p -> apps]: Passed regex - Collector is not catching unicode wrappers anymore" PingdomAdapter.reformatMapParams(nvPairs) must beFailing(NonEmptyList(expected)) } - def e5 = { + def e3 = { val querystring = toNameValuePairs( - "p" -> "apps", + "p" -> "apps", "message" -> """{"check": "1421338", "checkname": "Webhooks_Test", "host": "7eef51c2.ngrok.com", "action": "assign", "incidentid": 3, "description": "down"}""" ) - val payload = CollectorPayload(Shared.api, querystring, None, None, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, querystring, None, None, Shared.cljSource, Shared.context) val expected = RawEvent( Shared.api, Map( - "tv" -> "com.pingdom-v1", - "e" -> "ue", - "p" -> "apps", + "tv" -> "com.pingdom-v1", + "e" -> "ue", + "p" -> "apps", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.pingdom/incident_assign/jsonschema/1-0-0","data":{"check":"1421338","checkname":"Webhooks_Test","host":"7eef51c2.ngrok.com","incidentid":3,"description":"down"}}}""" ), None, @@ -100,16 +90,18 @@ class PingdomAdapterSpec extends Specification with DataTables with ValidationMa PingdomAdapter.toRawEvents(payload) must beSuccessful(NonEmptyList(expected)) } - def e6 = { - val payload = CollectorPayload(Shared.api, Nil, None, None, Shared.cljSource, Shared.context) + def e4 = { + val payload = CollectorPayload(Shared.api, Nil, None, None, Shared.cljSource, Shared.context) val expected = "Pingdom payload querystring is empty: nothing to process" PingdomAdapter.toRawEvents(payload) must beFailing(NonEmptyList(expected)) } - def e7 = { + def e5 = { val querystring = toNameValuePairs("p" -> "apps") - val payload = CollectorPayload(Shared.api, querystring, None, None, Shared.cljSource, Shared.context) - val expected = "Pingdom payload querystring does not have 'message' as a key: no event to process" + val payload = + CollectorPayload(Shared.api, querystring, None, None, Shared.cljSource, Shared.context) + val expected = + "Pingdom payload querystring does not have 'message' as a key" PingdomAdapter.toRawEvents(payload) must beFailing(NonEmptyList(expected)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala index ff4049b14..8e81e3915 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala @@ -19,24 +19,22 @@ import org.specs2.mutable.Specification import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import loaders._ class SendgridAdapterSpec extends Specification with ValidationMatchers { - implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.sendgrid", "v3") + val api = CollectorApi("com.sendgrid", "v3") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/json" @@ -214,7 +212,13 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { "toRawEvents" should { val payload = - CollectorPayload(Shared.api, Nil, ContentType.some, samplePostPayload.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + Nil, + ContentType.some, + samplePostPayload.some, + Shared.cljSource, + Shared.context) val actual = SendgridAdapter.toRawEvents(payload) "return the correct number of events" in { @@ -227,7 +231,7 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { "have the correct api endpoint for each element" in { actual must beSuccessful val items = actual.toList.head.toList - val siz = items.count(itm => itm.api == Shared.api) + val siz = items.count(itm => itm.api == Shared.api) siz must beEqualTo(items.size) } @@ -235,7 +239,7 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { "have the correct content type for each element" in { actual must beSuccessful val items = actual.toList.head.toList - val siz = items.count(itm => itm.contentType.get == ContentType) + val siz = items.count(itm => itm.contentType.get == ContentType) siz must beEqualTo(items.toList.size) } @@ -243,7 +247,7 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { "have the correct source for each element" in { actual must beSuccessful val items = actual.toList.head.toList - val siz = items.count(itm => itm.source == Shared.cljSource) + val siz = items.count(itm => itm.source == Shared.cljSource) siz must beEqualTo(items.toList.size) } @@ -251,44 +255,53 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { "have the correct context for each element" in { actual must beSuccessful val items = actual.toList.head.toList - val siz = items.count(itm => itm.context == Shared.context) + val siz = items.count(itm => itm.context == Shared.context) siz must beEqualTo(items.toList.size) } "reject empty bodies" in { - val invalidpayload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) - val toBeRejected = SendgridAdapter.toRawEvents(invalidpayload) + val invalidpayload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + val toBeRejected = SendgridAdapter.toRawEvents(invalidpayload) toBeRejected must beFailing } "reject empty content type" in { val invalidpayload = - CollectorPayload(Shared.api, Nil, None, samplePostPayload.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + Nil, + None, + samplePostPayload.some, + Shared.cljSource, + Shared.context) val toBeRejected = SendgridAdapter.toRawEvents(invalidpayload) toBeRejected must beFailing } "reject unexpected content type" in { val invalidpayload = - CollectorPayload(Shared.api, - Nil, - "invalidtype/invalid".some, - samplePostPayload.some, - Shared.cljSource, - Shared.context) + CollectorPayload( + Shared.api, + Nil, + "invalidtype/invalid".some, + samplePostPayload.some, + Shared.cljSource, + Shared.context) SendgridAdapter.toRawEvents(invalidpayload) must beFailing } "accept content types with explicit charsets" in { val payload = - CollectorPayload(Shared.api, - Nil, - "application/json; charset=utf-8".some, - samplePostPayload.some, - Shared.cljSource, - Shared.context) + CollectorPayload( + Shared.api, + Nil, + "application/json; charset=utf-8".some, + samplePostPayload.some, + Shared.cljSource, + Shared.context) val res = SendgridAdapter.toRawEvents(payload) res must beSuccessful } @@ -310,30 +323,39 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { ]""" val invalidpayload = - CollectorPayload(Shared.api, Nil, ContentType.some, invalidEventTypeJson.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + Nil, + ContentType.some, + invalidEventTypeJson.some, + Shared.cljSource, + Shared.context) SendgridAdapter.toRawEvents(invalidpayload) must beFailing } "reject invalid/unparsable json" in { val unparsableJson = """[ """ - SendgridAdapter.toRawEvents(CollectorPayload(Shared.api, - Nil, - ContentType.some, - unparsableJson.some, - Shared.cljSource, - Shared.context)) must beFailing + SendgridAdapter.toRawEvents( + CollectorPayload( + Shared.api, + Nil, + ContentType.some, + unparsableJson.some, + Shared.cljSource, + Shared.context)) must beFailing } "reject valid json in incorrect format" in { val incorrectlyFormattedJson = """[ ]""" SendgridAdapter.toRawEvents( - CollectorPayload(Shared.api, - Nil, - ContentType.some, - incorrectlyFormattedJson.some, - Shared.cljSource, - Shared.context)) must beFailing + CollectorPayload( + Shared.api, + Nil, + ContentType.some, + incorrectlyFormattedJson.some, + Shared.cljSource, + Shared.context)) must beFailing } "reject a payload with a some valid, some invalid events" in { @@ -360,10 +382,16 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { ]""" val payload = - CollectorPayload(Shared.api, Nil, ContentType.some, missingEventType.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + Nil, + ContentType.some, + missingEventType.some, + Shared.cljSource, + Shared.context) val actual = SendgridAdapter.toRawEvents(payload) - actual must beFailing( - NonEmptyList("Sendgrid event at index [1] failed: type parameter not provided - cannot determine event type")) + actual must beFailing(NonEmptyList( + "Sendgrid event at index [1] failed: type parameter not provided - cannot determine event type")) } "return correct json for sample event, including stripping out event keypair and fixing timestamp" in { @@ -387,30 +415,16 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { ]""" val payload = - CollectorPayload(Shared.api, Nil, ContentType.some, inputJson.some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + Nil, + ContentType.some, + inputJson.some, + Shared.cljSource, + Shared.context) val expectedJson = - compact( - parse("""{ - "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", - "data":{ - "schema":"iglu:com.sendgrid/processed/jsonschema/2-0-0", - "data":{ - "email": "example@test.com", - "timestamp": "2015-11-03T11:20:15.000Z", - "smtp-id": "\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e", - "category": "cat facts", - "sg_event_id": "sZROwMGMagFgnOEmSdvhig==", - "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0", - "marketing_campaign_id":12345, - "marketing_campaign_name":"campaign name", - "marketing_campaign_version":"B", - "marketing_campaign_split_id":13471 - } - } - } - }""") - ) + """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.sendgrid/processed/jsonschema/2-0-0","data":{"email":"example@test.com","timestamp":"2015-11-03T11:20:15.000Z","smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e","category":"cat facts","sg_event_id":"sZROwMGMagFgnOEmSdvhig==","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0","marketing_campaign_id":12345,"marketing_campaign_name":"campaign name","marketing_campaign_version":"B","marketing_campaign_split_id":13471}}}}""" val actual = SendgridAdapter.toRawEvents(payload) actual must beSuccessful( @@ -418,9 +432,9 @@ class SendgridAdapterSpec extends Specification with ValidationMatchers { RawEvent( Shared.api, Map( - "tv" -> "com.sendgrid-v3", - "e" -> "ue", - "p" -> "srv", + "tv" -> "com.sendgrid-v3", + "e" -> "ue", + "p" -> "srv", "ue_pr" -> expectedJson // NB this includes removing the "event" keypair as redundant ), ContentType.some, diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala index 2b83364d0..973336284 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala @@ -23,7 +23,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} -class StatusGatorAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class StatusGatorAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the StatusgatorAdapter functionality toRawEvents must return a Success Nel if every event in the payload is successful $e1 @@ -37,14 +41,15 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validati implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.statusgator", "v1") + val api = CollectorApi("com.statusgator", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/x-www-form-urlencoded" @@ -52,7 +57,13 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validati def e1 = { val body = "service_name=CloudFlare&favicon_url=https%3A%2F%2Fdwxjd9cd6rwno.cloudfront.net%2Ffavicons%2Fcloudflare.ico&status_page_url=https%3A%2F%2Fwww.cloudflarestatus.com%2F&home_page_url=http%3A%2F%2Fwww.cloudflare.com¤t_status=up&last_status=warn&occurred_at=2016-05-19T09%3A26%3A31%2B00%3A00" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """|{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -71,16 +82,18 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validati |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.statusgator-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.statusgator-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) StatusGatorAdapter.toRawEvents(payload) must beSuccessful(expected) } def e2 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) StatusGatorAdapter.toRawEvents(payload) must beFailing( NonEmptyList("Request body is empty: no StatusGator events to process")) } @@ -88,25 +101,31 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validati def e3 = { val body = "service_name=CloudFlare&favicon_url=https%3A%2F%2Fdwxjd9cd6rwno.cloudfront.net%2Ffavicons%2Fcloudflare.ico&status_page_url=https%3A%2F%2Fwww.cloudflarestatus.com%2F&home_page_url=http%3A%2F%2Fwww.cloudflare.com¤t_status=up&last_status=warn&occurred_at=2016-05-19T09%3A26%3A31%2B00%3A00" - val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) - StatusGatorAdapter.toRawEvents(payload) must beFailing( - NonEmptyList( - "Request body provided but content type empty, expected application/x-www-form-urlencoded for StatusGator")) + val payload = + CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) + StatusGatorAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Request body provided but content type empty, expected application/x-www-form-urlencoded for StatusGator")) } def e4 = { val body = "service_name=CloudFlare&favicon_url=https%3A%2F%2Fdwxjd9cd6rwno.cloudfront.net%2Ffavicons%2Fcloudflare.ico&status_page_url=https%3A%2F%2Fwww.cloudflarestatus.com%2F&home_page_url=http%3A%2F%2Fwww.cloudflare.com¤t_status=up&last_status=warn&occurred_at=2016-05-19T09%3A26%3A31%2B00%3A00" - val ct = "application/json" - val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) - StatusGatorAdapter.toRawEvents(payload) must beFailing( - NonEmptyList( - "Content type of application/json provided, expected application/x-www-form-urlencoded for StatusGator")) + val ct = "application/json" + val payload = + CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) + StatusGatorAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Content type of application/json provided, expected application/x-www-form-urlencoded for StatusGator")) } def e5 = { - val body = "" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val body = "" + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("StatusGator event body is empty: nothing to process") StatusGatorAdapter.toRawEvents(payload) must beFailing(expected) } @@ -114,7 +133,13 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validati def e6 = { val body = "{service_name=CloudFlare&favicon_url=https%3A%2F%2Fdwxjd9cd6rwno.cloudfront.net%2Ffavicons%2Fcloudflare.ico&status_page_url=https%3A%2F%2Fwww.cloudflarestatus.com%2F&home_page_url=http%3A%2F%2Fwww.cloudflare.com¤t_status=up&last_status=warn&occurred_at=2016-05-19T09%3A26%3A31%2B00%3A00" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( "StatusGator incorrect event string : [Illegal character in query at index 18: http://localhost/?{service_name=CloudFlare&favicon_url=https%3A%2F%2Fdwxjd9cd6rwno.cloudfront.net%2Ffavicons%2Fcloudflare.ico&status_page_url=https%3A%2F%2Fwww.cloudflarestatus.com%2F&home_page_url=http%3A%2F%2Fwww.cloudflare.com¤t_status=up&last_status=warn&occurred_at=2016-05-19T09%3A26%3A31%2B00%3A00]") StatusGatorAdapter.toRawEvents(payload) must beFailing(expected) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala index 10d4249c2..280217989 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala @@ -24,7 +24,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} import SpecHelpers._ -class UnbounceAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class UnbounceAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the UnbounceAdapter functionality toRawEvents must return a Success Nel if the query string is valid $e1 @@ -44,14 +48,15 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.unbounce", "v1") + val api = CollectorApi("com.unbounce", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/x-www-form-urlencoded" @@ -60,7 +65,13 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_url=http%3A%2F%2Funbouncepages.com%2Fwayfaring-147%2F&page_name=Wayfaring&page_id=7648177d-7323-4330-b4f9-9951a52138b6&variant=a&data.json=%7B%22userfield1%22%3A%5B%22asdfasdf%22%5D%2C%22ip_address%22%3A%5B%2285.73.39.163%22%5D%2C%22page_uuid%22%3A%5B%227648177d-7323-4330-b4f9-9951a52138b6%22%5D%2C%22variant%22%3A%5B%22a%22%5D%2C%22time_submitted%22%3A%5B%2211%3A45+AM+UTC%22%5D%2C%22date_submitted%22%3A%5B%222017-11-15%22%5D%2C%22page_url%22%3A%5B%22http%3A%2F%2Funbouncepages.com%2Fwayfaring-147%2F%22%5D%2C%22page_name%22%3A%5B%22Wayfaring%22%5D%7D&data.xml=%3C%3Fxml+version%3D%221.0%22+encoding%3D%22UTF-8%22%3F%3E%3Cform_data%3E%3Cuserfield1%3Easdfasdf%3C%2Fuserfield1%3E%3Cip_address%3E85.73.39.163%3C%2Fip_address%3E%3Cpage_uuid%3E7648177d-7323-4330-b4f9-9951a52138b6%3C%2Fpage_uuid%3E%3Cvariant%3Ea%3C%2Fvariant%3E%3Ctime_submitted%3E11%3A45+AM+UTC%3C%2Ftime_submitted%3E%3Cdate_submitted%3E2017-11-15%3C%2Fdate_submitted%3E%3Cpage_url%3Ehttp%3A%2F%2Funbouncepages.com%2Fwayfaring-147%2F%3C%2Fpage_url%3E%3Cpage_name%3EWayfaring%3C%2Fpage_name%3E%3C%2Fform_data%3E" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expectedJson = """{ |"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -101,17 +112,19 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM |} |}""".stripMargin.replaceAll("[\n\r]", "") val expected = NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.unbounce-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context)) + RawEvent( + Shared.api, + Map("tv" -> "com.unbounce-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context)) UnbounceAdapter.toRawEvents(payload) must beSuccessful(expected) } def e2 = { - val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") - val payload = CollectorPayload(Shared.api, params, ContentType.some, None, Shared.cljSource, Shared.context) + val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") + val payload = + CollectorPayload(Shared.api, params, ContentType.some, None, Shared.cljSource, Shared.context) UnbounceAdapter.toRawEvents(payload) must beFailing( NonEmptyList("Request body is empty: no Unbounce events to process")) } @@ -119,26 +132,32 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM def e3 = { val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&page_name=Test-Webhook&variant=a&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1&data.json=%7B%22email%22%3A%5B%22test%40snowplowanalytics.com%22%5D%2C%22ip_address%22%3A%5B%22200.121.220.179%22%5D%2C%22time_submitted%22%3A%5B%2204%3A17%20PM%20UTC%22%5D%7D" - val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) - UnbounceAdapter.toRawEvents(payload) must beFailing( - NonEmptyList( - "Request body provided but content type empty, expected application/x-www-form-urlencoded for Unbounce")) + val payload = + CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) + UnbounceAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Request body provided but content type empty, expected application/x-www-form-urlencoded for Unbounce")) } def e4 = { val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&page_name=Test-Webhook&variant=a&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1&data.json=%7B%22email%22%3A%5B%22test%40snowplowanalytics.com%22%5D%2C%22ip_address%22%3A%5B%22200.121.220.179%22%5D%2C%22time_submitted%22%3A%5B%2204%3A17%20PM%20UTC%22%5D%7D" - val ct = "application/json" - val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) - UnbounceAdapter.toRawEvents(payload) must beFailing( - NonEmptyList( - "Content type of application/json provided, expected application/x-www-form-urlencoded for Unbounce")) + val ct = "application/json" + val payload = + CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) + UnbounceAdapter.toRawEvents(payload) must beFailing(NonEmptyList( + "Content type of application/json provided, expected application/x-www-form-urlencoded for Unbounce")) } def e5 = { - val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") - val body = "" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") + val body = "" + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Unbounce event body is empty: nothing to process") UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } @@ -147,7 +166,13 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&page_name=Test-Webhook&variant=a&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Unbounce event data does not have 'data.json' as a key") UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } @@ -156,7 +181,13 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&page_name=Test-Webhook&variant=a&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1&data.json=" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Unbounce event data is empty: nothing to process") UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } @@ -165,9 +196,16 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&page_name=Test-Webhook&variant=a&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1&data.json=%7B%7B%22email%22%3A%5B%22test%40snowplowanalytics.com%22%5D%2C%22ip_address%22%3A%5B%22200.121.220.179%22%5D%2C%22time_submitted%22%3A%5B%2204%3A17%20PM%20UTC%22%5D%7D" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( - "Unbounce event string failed to parse into JSON: [Unexpected character ('{' (code 123)): was expecting double-quote to start field name at [Source: (String)\"{{\"email\":[\"test@snowplowanalytics.com\"],\"ip_address\":[\"200.121.220.179\"],\"time_submitted\":[\"04:17 PM UTC\"]}\"; line: 1, column: 3]]") + """Unbounce event string failed to parse into JSON: [expected " got '{"emai...' (line 1, column 2)]""" + ) UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } @@ -175,7 +213,13 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_name=Test-Webhook&variant=a&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1&data.json=%7B%22email%22%3A%5B%22test%40snowplowanalytics.com%22%5D%2C%22ip_address%22%3A%5B%22200.121.220.179%22%5D%2C%22time_submitted%22%3A%5B%2204%3A17%20PM%20UTC%22%5D%7D" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Unbounce context data missing 'page_id'") UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } @@ -184,7 +228,13 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&variant=a&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1&data.json=%7B%22email%22%3A%5B%22test%40snowplowanalytics.com%22%5D%2C%22ip_address%22%3A%5B%22200.121.220.179%22%5D%2C%22time_submitted%22%3A%5B%2204%3A17%20PM%20UTC%22%5D%7D" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Unbounce context data missing 'page_name'") UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } @@ -193,7 +243,13 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&page_name=Test-Webhook&page_url=http%3A%2F%2Funbouncepages.com%2Ftest-webhook-1&data.json=%7B%22email%22%3A%5B%22test%40snowplowanalytics.com%22%5D%2C%22ip_address%22%3A%5B%22200.121.220.179%22%5D%2C%22time_submitted%22%3A%5B%2204%3A17%20PM%20UTC%22%5D%7D" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Unbounce context data missing 'variant'") UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } @@ -202,7 +258,13 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidationM val params = toNameValuePairs("schema" -> "iglu:com.unbounce/test/jsonschema/1-0-0") val body = "page_id=f7afd389-65a3-45fa-8bad-b7a42236044c&page_name=Test-Webhook&variant=a&data.json=%7B%22email%22%3A%5B%22test%40snowplowanalytics.com%22%5D%2C%22ip_address%22%3A%5B%22200.121.220.179%22%5D%2C%22time_submitted%22%3A%5B%2204%3A17%20PM%20UTC%22%5D%7D" - val payload = CollectorPayload(Shared.api, params, ContentType.some, body.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + params, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList("Unbounce context data missing 'page_url'") UnbounceAdapter.toRawEvents(payload) must beFailing(expected) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala index 912dd0bda..f19dcec15 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala @@ -13,86 +13,84 @@ package com.snowplowanalytics.snowplow.enrich.common package adapters.registry +import io.circe.literal._ +import io.circe.parser._ import org.joda.time.DateTime -import scalaz.Scalaz._ -import scalaz._ import org.specs2.mutable.Specification import org.specs2.scalaz.ValidationMatchers -import org.json4s._ -import org.json4s.jackson.JsonMethods._ +import scalaz.Scalaz._ +import scalaz._ import loaders._ class UrbanAirshipAdapterSpec extends Specification with ValidationMatchers { implicit val resolver = SpecHelpers.IgluResolver - implicit val formats = DefaultFormats object Shared { - val api = CollectorApi("com.urbanairship.connect", "v1") + val api = CollectorApi("com.urbanairship.connect", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(None, "37.157.33.123".some, None, None, Nil, None) // NB the collector timestamp is set to None! + val context = CollectorContext(None, "37.157.33.123".some, None, None, Nil, None) // NB the collector timestamp is set to None! } "toRawEvents" should { - val validPayload = """ - |{ - | "id": "e3314efb-9058-dbaf-c4bb-b754fca73613", - | "offset": "1", - | "occurred": "2015-11-13T16:31:52.393Z", - | "processed": "2015-11-13T16:31:52.393Z", - | "device": { - | "amazon_channel": "cd97c95c-ed77-f15a-3a67-5c2e26799d35" - | }, - | "body": { - | "session_id": "27c75cab-a0b8-9da2-bc07-6d7253e0e13f" - | }, - | "type": "CLOSE" - |} - |""".stripMargin - - val invalidEvent = """ - |{ - | "id": "e3314efb-9058-dbaf-c4bb-b754fca73613", - | "offset": "1", - | "occurred": "2015-11-13T16:31:52.393Z", - | "processed": "2015-11-13T16:31:52.393Z", - | "device": { - | "amazon_channel": "cd97c95c-ed77-f15a-3a67-5c2e26799d35" - | }, - | "body": { - | "session_id": "27c75cab-a0b8-9da2-bc07-6d7253e0e13f" - | }, - | "type": "NOT_AN_EVENT_TYPE" - |} - |""".stripMargin - - val payload = CollectorPayload(Shared.api, Nil, None, validPayload.some, Shared.cljSource, Shared.context) - val actual = UrbanAirshipAdapter.toRawEvents(payload) - - val expectedUnstructEventJson = """|{ - | "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", - | "data":{ - | "schema":"iglu:com.urbanairship.connect/CLOSE/jsonschema/1-0-0", - | "data":{ - | "id": "e3314efb-9058-dbaf-c4bb-b754fca73613", - | "offset": "1", - | "occurred": "2015-11-13T16:31:52.393Z", - | "processed": "2015-11-13T16:31:52.393Z", - | "device": { - | "amazon_channel": "cd97c95c-ed77-f15a-3a67-5c2e26799d35" - | }, - | "body": { - | "session_id": "27c75cab-a0b8-9da2-bc07-6d7253e0e13f" - | }, - | "type": "CLOSE" - | } - | } - |} - """.stripMargin - - val expectedCompactedUnstructEvent = compact(parse(expectedUnstructEventJson)) + val validPayload = json"""{ + "id": "e3314efb-9058-dbaf-c4bb-b754fca73613", + "offset": "1", + "occurred": "2015-11-13T16:31:52.393Z", + "processed": "2015-11-13T16:31:52.393Z", + "device": { + "amazon_channel": "cd97c95c-ed77-f15a-3a67-5c2e26799d35" + }, + "body": { + "session_id": "27c75cab-a0b8-9da2-bc07-6d7253e0e13f" + }, + "type": "CLOSE" + }""" + + val invalidEvent = json"""{ + "id": "e3314efb-9058-dbaf-c4bb-b754fca73613", + "offset": "1", + "occurred": "2015-11-13T16:31:52.393Z", + "processed": "2015-11-13T16:31:52.393Z", + "device": { + "amazon_channel": "cd97c95c-ed77-f15a-3a67-5c2e26799d35" + }, + "body": { + "session_id": "27c75cab-a0b8-9da2-bc07-6d7253e0e13f" + }, + "type": "NOT_AN_EVENT_TYPE" + }""" + + val payload = CollectorPayload( + Shared.api, + Nil, + None, + validPayload.noSpaces.some, + Shared.cljSource, + Shared.context) + val actual = UrbanAirshipAdapter.toRawEvents(payload) + + val expectedUnstructEventJson = json"""{ + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data":{ + "schema":"iglu:com.urbanairship.connect/CLOSE/jsonschema/1-0-0", + "data":{ + "id": "e3314efb-9058-dbaf-c4bb-b754fca73613", + "offset": "1", + "occurred": "2015-11-13T16:31:52.393Z", + "processed": "2015-11-13T16:31:52.393Z", + "device": { + "amazon_channel": "cd97c95c-ed77-f15a-3a67-5c2e26799d35" + }, + "body": { + "session_id": "27c75cab-a0b8-9da2-bc07-6d7253e0e13f" + }, + "type": "CLOSE" + } + } + }""" "return the correct number of events (1)" in { actual must beSuccessful @@ -102,36 +100,57 @@ class UrbanAirshipAdapterSpec extends Specification with ValidationMatchers { "link to the correct json schema for the event type" in { actual must beSuccessful - val correctType = (parse(validPayload) \ "type").extract[String] - correctType must be equalTo ("CLOSE") + val correctType = validPayload.hcursor.get[String]("type") + correctType must be equalTo (Right("CLOSE")) - val items = actual.toList.head.toList - val sentSchema = (parse(items.head.parameters("ue_pr")) \ "data") \ "schema" - sentSchema.extract[String] must beEqualTo("""iglu:com.urbanairship.connect/CLOSE/jsonschema/1-0-0""") + val items = actual.toList.head.toList + val sentSchema = parse(items.head.parameters("ue_pr")) + .leftMap(_.getMessage) + .flatMap(_.hcursor.downField("data").get[String]("schema").leftMap(_.getMessage)) + sentSchema must beRight("""iglu:com.urbanairship.connect/CLOSE/jsonschema/1-0-0""") } "fail on unknown event types" in { - val payload = CollectorPayload(Shared.api, Nil, None, invalidEvent.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + None, + invalidEvent.noSpaces.some, + Shared.cljSource, + Shared.context) UrbanAirshipAdapter.toRawEvents(payload) must beFailing } "reject unparsable json" in { - val payload = CollectorPayload(Shared.api, Nil, None, """{ """.some, Shared.cljSource, Shared.context) + val payload = + CollectorPayload(Shared.api, Nil, None, """{ """.some, Shared.cljSource, Shared.context) UrbanAirshipAdapter.toRawEvents(payload) must beFailing } "reject badly formatted json" in { val payload = - CollectorPayload(Shared.api, Nil, None, """{ "value": "str" }""".some, Shared.cljSource, Shared.context) + CollectorPayload( + Shared.api, + Nil, + None, + """{ "value": "str" }""".some, + Shared.cljSource, + Shared.context) UrbanAirshipAdapter.toRawEvents(payload) must beFailing } "reject content types" in { - val payload = - CollectorPayload(Shared.api, Nil, "a/type".some, validPayload.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + "a/type".some, + validPayload.noSpaces.some, + Shared.cljSource, + Shared.context) val res = UrbanAirshipAdapter.toRawEvents(payload) - res must beFailing(NonEmptyList("Content type of a/type provided, expected None for UrbanAirship")) + res must beFailing( + NonEmptyList("Content type of a/type provided, expected None for UrbanAirship")) } "populate content-type as None (it's not applicable)" in { @@ -147,35 +166,37 @@ class UrbanAirshipAdapterSpec extends Specification with ValidationMatchers { "have the correct context, including setting the correct collector timestamp" in { val context = actual.getOrElse(throw new IllegalStateException).head.context Shared.context.timestamp mustEqual None - context mustEqual Shared.context.copy(timestamp = DateTime.parse("2015-11-13T16:31:52.393Z").some) // it should be set to the "processed" field by the adapter + context mustEqual Shared.context.copy( + timestamp = DateTime + .parse("2015-11-13T16:31:52.393Z") + .some) // it should be set to the "processed" field by the adapter } "return the correct unstruct_event json" in { actual match { - case Success(successes) => { + case Success(successes) => val event = successes.head - compact(parse(event.parameters("ue_pr"))) must beEqualTo(expectedCompactedUnstructEvent) - } + parse(event.parameters("ue_pr")) must beRight(expectedUnstructEventJson) case _ => ko("payload was not accepted") } } "correctly populate the true timestamp" in { actual match { - case Success(successes) => { + case Success(successes) => val event = successes.head - event.parameters("ttm") must beEqualTo("1447432312393") // "occurred" field value in ms past epoch (2015-11-13T16:31:52.393Z) - } + // "occurred" field value in ms past epoch (2015-11-13T16:31:52.393Z) + event.parameters("ttm") must beEqualTo("1447432312393") case _ => ko("payload was not populated") } } "correctly populate the eid" in { actual match { - case Success(successes) => { + case Success(successes) => val event = successes.head - event.parameters("eid") must beEqualTo("e3314efb-9058-dbaf-c4bb-b754fca73613") // id field value - } + // id field value + event.parameters("eid") must beEqualTo("e3314efb-9058-dbaf-c4bb-b754fca73613") case _ => ko("payload was not populated") } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala index ff5feda6f..b67a8c6b4 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala @@ -23,7 +23,11 @@ import Scalaz._ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} -class VeroAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class VeroAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the VeroAdapter functionality toRawEvents must return a success for a valid "sent" type payload body being passed $e1 @@ -41,14 +45,15 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch implicit val resolver = SpecHelpers.IgluResolver object Shared { - val api = CollectorApi("com.getvero", "v1") + val api = CollectorApi("com.getvero", "v1") val cljSource = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2018-01-01T00:00:00.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2018-01-01T00:00:00.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } val ContentType = "application/json" @@ -56,15 +61,21 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e1 = { val bodyStr = """{"sent_at": 1435016238, "event": {"name": "Test event", "triggered_at": 1424012238}, "type": "sent", "user": {"id": 123, "email": "steve@getvero.com"},"campaign": {"id": 987, "type": "transactional", "name": "Order confirmation", "subject": "Your order is being processed", "trigger-event": "purchased item", "permalink": "http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25", "variation": "Variation A", "tags": "tag 1, tag 2"}}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/sent/jsonschema/1-0-0","data":{"sent_at":"2015-06-22T23:37:18.000Z","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"user":{"id":123,"email":"steve@getvero.com"},"campaign":{"id":987,"type":"transactional","name":"Order confirmation","subject":"Your order is being processed","trigger-event":"purchased item","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","variation":"Variation A","tags":"tag 1, tag 2"}}}}""" + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/sent/jsonschema/1-0-0","data":{"event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"sent_at":"2015-06-22T23:37:18.000Z","campaign":{"name":"Order confirmation","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","subject":"Your order is being processed","tags":"tag 1, tag 2","variation":"Variation A","trigger-event":"purchased item","id":987,"type":"transactional"},"user":{"id":123,"email":"steve@getvero.com"}}}}""" ), ContentType.some, Shared.cljSource, @@ -76,15 +87,21 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e2 = { val bodyStr = """{"delivered_at": 1435016238, "sender_ip": "127.0.0.1", "message_id": "20130920062934.21270.53268@vero.com", "event":{"name":"Test event","triggered_at":1424012238}, "type": "delivered", "user": {"id": 123, "email": "steve@getvero.com"},"campaign": {"id": 987, "type": "transactional", "name": "Order confirmation", "subject": "Your order is being processed", "trigger-event": "purchased item", "permalink": "http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25", "variation": "Variation A", "tags": "tag 1, tag 2"}}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/delivered/jsonschema/1-0-0","data":{"delivered_at":"2015-06-22T23:37:18.000Z","sender_ip":"127.0.0.1","message_id":"20130920062934.21270.53268@vero.com","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"user":{"id":123,"email":"steve@getvero.com"},"campaign":{"id":987,"type":"transactional","name":"Order confirmation","subject":"Your order is being processed","trigger-event":"purchased item","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","variation":"Variation A","tags":"tag 1, tag 2"}}}}""" + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/delivered/jsonschema/1-0-0","data":{"event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"campaign":{"name":"Order confirmation","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","subject":"Your order is being processed","tags":"tag 1, tag 2","variation":"Variation A","trigger-event":"purchased item","id":987,"type":"transactional"},"delivered_at":"2015-06-22T23:37:18.000Z","message_id":"20130920062934.21270.53268@vero.com","sender_ip":"127.0.0.1","user":{"id":123,"email":"steve@getvero.com"}}}}""" ), ContentType.some, Shared.cljSource, @@ -96,15 +113,21 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e3 = { val bodyStr = """{"opened_at": 1435016238, "user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", "message_id": "20130920062934.21270.53268@vero.com", "event": {"name": "Test event", "triggered_at": 1424012238}, "type": "opened", "user": {"id": 123, "email": "steve@getvero.com"},"campaign": {"id": 987, "type": "transactional", "name": "Order confirmation", "subject": "Your order is being processed", "trigger-event": "purchased item", "permalink": "http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25", "variation": "Variation A", "tags": "tag 1, tag 2"}}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/opened/jsonschema/1-0-0","data":{"opened_at":"2015-06-22T23:37:18.000Z","user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)","message_id":"20130920062934.21270.53268@vero.com","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"user":{"id":123,"email":"steve@getvero.com"},"campaign":{"id":987,"type":"transactional","name":"Order confirmation","subject":"Your order is being processed","trigger-event":"purchased item","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","variation":"Variation A","tags":"tag 1, tag 2"}}}}""" + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/opened/jsonschema/1-0-0","data":{"opened_at":"2015-06-22T23:37:18.000Z","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"campaign":{"name":"Order confirmation","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","subject":"Your order is being processed","tags":"tag 1, tag 2","variation":"Variation A","trigger-event":"purchased item","id":987,"type":"transactional"},"user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)","message_id":"20130920062934.21270.53268@vero.com","user":{"id":123,"email":"steve@getvero.com"}}}}""" ), ContentType.some, Shared.cljSource, @@ -116,15 +139,21 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e4 = { val bodyStr = """{"clicked_at": 1435016238, "user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", "message_id": "20130920062934.21270.53268@vero.com", "event": {"name": "Test event", "triggered_at": 1424012238}, "type": "clicked", "user": {"id": 123, "email": "steve@getvero.com"},"campaign": {"id": 987, "type": "transactional", "name": "Order confirmation", "subject": "Your order is being processed", "trigger-event": "purchased item", "permalink": "http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25", "variation": "Variation A", "tags": "tag 1, tag 2"}}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/clicked/jsonschema/1-0-0","data":{"clicked_at":"2015-06-22T23:37:18.000Z","user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)","message_id":"20130920062934.21270.53268@vero.com","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"user":{"id":123,"email":"steve@getvero.com"},"campaign":{"id":987,"type":"transactional","name":"Order confirmation","subject":"Your order is being processed","trigger-event":"purchased item","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","variation":"Variation A","tags":"tag 1, tag 2"}}}}""" + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/clicked/jsonschema/1-0-0","data":{"clicked_at":"2015-06-22T23:37:18.000Z","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"campaign":{"name":"Order confirmation","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","subject":"Your order is being processed","tags":"tag 1, tag 2","variation":"Variation A","trigger-event":"purchased item","id":987,"type":"transactional"},"user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)","message_id":"20130920062934.21270.53268@vero.com","user":{"id":123,"email":"steve@getvero.com"}}}}""" ), ContentType.some, Shared.cljSource, @@ -136,15 +165,21 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e5 = { val bodyStr = """{"bounced_at": 1435016238, "bounce_type":"hard", "bounce_code": "521", "bounce_message": "521 5.2.1 : AOL will not accept delivery of this message.", "message_id": "20130920062934.21270.53268@vero.com", "event": {"name": "Test event", "triggered_at": 1424012238}, "type": "bounced", "user": {"id": 123, "email": "steve@getvero.com"},"campaign": {"id": 987, "type": "transactional", "name": "Order confirmation", "subject": "Your order is being processed", "trigger-event": "purchased item", "permalink": "http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25", "variation": "Variation A"}}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/bounced/jsonschema/1-0-0","data":{"bounced_at":"2015-06-22T23:37:18.000Z","bounce_type":"hard","bounce_code":"521","bounce_message":"521 5.2.1 : AOL will not accept delivery of this message.","message_id":"20130920062934.21270.53268@vero.com","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"user":{"id":123,"email":"steve@getvero.com"},"campaign":{"id":987,"type":"transactional","name":"Order confirmation","subject":"Your order is being processed","trigger-event":"purchased item","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","variation":"Variation A"}}}}""" + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/bounced/jsonschema/1-0-0","data":{"bounce_message":"521 5.2.1 : AOL will not accept delivery of this message.","event":{"name":"Test event","triggered_at":"2015-02-15T14:57:18.000Z"},"bounced_at":"2015-06-22T23:37:18.000Z","campaign":{"name":"Order confirmation","permalink":"http://app.getvero.com/view/1/341d64944577ac1f70f560e37db54a25","subject":"Your order is being processed","variation":"Variation A","trigger-event":"purchased item","id":987,"type":"transactional"},"message_id":"20130920062934.21270.53268@vero.com","bounce_type":"hard","bounce_code":"521","user":{"id":123,"email":"steve@getvero.com"}}}}""" ), ContentType.some, Shared.cljSource, @@ -156,14 +191,20 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e6 = { val bodyStr = """{"unsubscribed_at": 1435016238, "type": "unsubscribed", "user": {"id": 123, "email": "steve@getvero.com"}}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/unsubscribed/jsonschema/1-0-0","data":{"unsubscribed_at":"2015-06-22T23:37:18.000Z","user":{"id":123,"email":"steve@getvero.com"}}}}""" ), ContentType.some, @@ -176,15 +217,21 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e7 = { val bodyStr = """{"type": "user_created", "user": {"id": 123, "email": "steve@getvero.com"}, "firstname": "Steve", "company": "Vero", "role": "Bot"}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", - "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/created/jsonschema/1-0-0","data":{"user":{"id":123,"email":"steve@getvero.com"},"firstname":"Steve","company":"Vero","role":"Bot"}}}""" + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", + "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/created/jsonschema/1-0-0","data":{"role":"Bot","firstname":"Steve","company":"Vero","user":{"id":123,"email":"steve@getvero.com"}}}}""" ), ContentType.some, Shared.cljSource, @@ -196,14 +243,20 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch def e8 = { val bodyStr = """{"type": "user_updated", "user": {"id": 123, "email": "steve@getvero.com"}, "changes": {"_tags": {"add": ["active-customer"], "remove": ["unactive-180-days"]}}}""" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, bodyStr.some, Shared.cljSource, Shared.context) + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + bodyStr.some, + Shared.cljSource, + Shared.context) val expected = NonEmptyList( RawEvent( Shared.api, Map( - "tv" -> "com.getvero-v1", - "e" -> "ue", - "p" -> "srv", + "tv" -> "com.getvero-v1", + "e" -> "ue", + "p" -> "srv", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.getvero/updated/jsonschema/1-0-0","data":{"user":{"id":123,"email":"steve@getvero.com"},"changes":{"tags":{"add":["active-customer"],"remove":["unactive-180-days"]}}}}}""" ), ContentType.some, @@ -214,30 +267,40 @@ class VeroAdapterSpec extends Specification with DataTables with ValidationMatch } def e9 = - "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED SCHEMA" | - "Valid, type sent" !! "sent" ! "iglu:com.getvero/sent/jsonschema/1-0-0" | + "SPEC NAME" || "SCHEMA TYPE" | "EXPECTED SCHEMA" | + "Valid, type sent" !! "sent" ! "iglu:com.getvero/sent/jsonschema/1-0-0" | "Valid, type unsubscribed" !! "unsubscribed" ! "iglu:com.getvero/unsubscribed/jsonschema/1-0-0" | - "Valid, type delivered" !! "delivered" ! "iglu:com.getvero/delivered/jsonschema/1-0-0" | - "Valid, type opened" !! "opened" ! "iglu:com.getvero/opened/jsonschema/1-0-0" | - "Valid, type clicked" !! "clicked" ! "iglu:com.getvero/clicked/jsonschema/1-0-0" | - "Valid, type created" !! "user_created" ! "iglu:com.getvero/created/jsonschema/1-0-0" | - "Valid, type updated" !! "user_updated" ! "iglu:com.getvero/updated/jsonschema/1-0-0" | - "Valid, type bounced" !! "bounced" ! "iglu:com.getvero/bounced/jsonschema/1-0-0" |> { (_, schema, expected) => - val body = "{\"type\":\"" + schema + "\"}" - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, body.some, Shared.cljSource, Shared.context) - val expectedJson = "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"" + expected + "\",\"data\":{}}}" - val actual = VeroAdapter.toRawEvents(payload) - actual must beSuccessful( - NonEmptyList( - RawEvent(Shared.api, - Map("tv" -> "com.getvero-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), - ContentType.some, - Shared.cljSource, - Shared.context))) + "Valid, type delivered" !! "delivered" ! "iglu:com.getvero/delivered/jsonschema/1-0-0" | + "Valid, type opened" !! "opened" ! "iglu:com.getvero/opened/jsonschema/1-0-0" | + "Valid, type clicked" !! "clicked" ! "iglu:com.getvero/clicked/jsonschema/1-0-0" | + "Valid, type created" !! "user_created" ! "iglu:com.getvero/created/jsonschema/1-0-0" | + "Valid, type updated" !! "user_updated" ! "iglu:com.getvero/updated/jsonschema/1-0-0" | + "Valid, type bounced" !! "bounced" ! "iglu:com.getvero/bounced/jsonschema/1-0-0" |> { + (_, schema, expected) => + val body = "{\"type\":\"" + schema + "\"}" + val payload = CollectorPayload( + Shared.api, + Nil, + ContentType.some, + body.some, + Shared.cljSource, + Shared.context) + val expectedJson = "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"" + expected + "\",\"data\":{}}}" + val actual = VeroAdapter.toRawEvents(payload) + actual must beSuccessful( + NonEmptyList( + RawEvent( + Shared.api, + Map("tv" -> "com.getvero-v1", "e" -> "ue", "p" -> "srv", "ue_pr" -> expectedJson), + ContentType.some, + Shared.cljSource, + Shared.context))) } def e10 = { - val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) - VeroAdapter.toRawEvents(payload) must beFailing(NonEmptyList("Request body is empty: no Vero event to process")) + val payload = + CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) + VeroAdapter.toRawEvents(payload) must beFailing( + NonEmptyList("Request body is empty: no Vero event to process")) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala index 7b11c2a5a..c49b5eb92 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala @@ -26,7 +26,11 @@ import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSourc import utils.{ConversionUtils => CU} import SpecHelpers._ -class SnowplowAdapterSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class SnowplowAdapterSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the SnowplowAdapter functionality Tp1.toRawEvents should return a NEL containing one RawEvent if the querystring is populated $e1 @@ -35,7 +39,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM Tp2.toRawEvents should return a NEL containing one RawEvent if the querystring is empty but the body contains one event $e4 Tp2.toRawEvents should return a NEL containing three RawEvents consolidating body's events and querystring's parameters $e5 Tp1.toRawEvents should return a NEL containing one RawEvent if the Content-Type is application/json; charset=UTF-8 $e6 - Tp2.toRawEvents should return a Validation Failure if querystring, body and content type are mismatching $e8 + Tp2.toRawEvents should return a Validation Failure if querystring, body and content type are mismatching $e7 Tp2.toRawEvents should return a Validation Failure if the body is not a self-describing JSON $e8 Tp2.toRawEvents should return a Validation Failure if the body is in a JSON Schema other than payload_data $e9 Tp2.toRawEvents should return a Validation Failure if the body fails payload_data JSON Schema validation $e10 @@ -53,78 +57,108 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM implicit val resolver = SpecHelpers.IgluResolver object Snowplow { - private val api: (String) => CollectorApi = version => CollectorApi("com.snowplowanalytics.snowplow", version) + private val api: (String) => CollectorApi = version => + CollectorApi("com.snowplowanalytics.snowplow", version) val Tp1 = api("tp1") val Tp2 = api("tp2") } - val ApplicationJson = "application/json" - val ApplicationJsonWithCharset = "application/json; charset=utf-8" + val ApplicationJson = "application/json" + val ApplicationJsonWithCharset = "application/json; charset=utf-8" val ApplicationJsonWithCapitalCharset = "application/json; charset=UTF-8" object Shared { val source = CollectorSource("clj-tomcat", "UTF-8", None) - val context = CollectorContext(DateTime.parse("2013-08-29T00:18:48.000+00:00").some, - "37.157.33.123".some, - None, - None, - Nil, - None) + val context = CollectorContext( + DateTime.parse("2013-08-29T00:18:48.000+00:00").some, + "37.157.33.123".some, + None, + None, + Nil, + None) } def e1 = { val payload = - CollectorPayload(Snowplow.Tp1, toNameValuePairs("aid" -> "test"), None, None, Shared.source, Shared.context) + CollectorPayload( + Snowplow.Tp1, + toNameValuePairs("aid" -> "test"), + None, + None, + Shared.source, + Shared.context) val actual = Tp1Adapter.toRawEvents(payload) actual must beSuccessful( - NonEmptyList(RawEvent(Snowplow.Tp1, Map("aid" -> "test"), None, Shared.source, Shared.context))) + NonEmptyList( + RawEvent(Snowplow.Tp1, Map("aid" -> "test"), None, Shared.source, Shared.context))) } def e2 = { val payload = CollectorPayload(Snowplow.Tp1, Nil, None, None, Shared.source, Shared.context) - val actual = Tp1Adapter.toRawEvents(payload) + val actual = Tp1Adapter.toRawEvents(payload) actual must beFailing(NonEmptyList("Querystring is empty: no raw event to process")) } def e3 = { - val payload = CollectorPayload(Snowplow.Tp2, - toNameValuePairs("aid" -> "tp2", "e" -> "se"), - None, - None, - Shared.source, - Shared.context) + val payload = CollectorPayload( + Snowplow.Tp2, + toNameValuePairs("aid" -> "tp2", "e" -> "se"), + None, + None, + Shared.source, + Shared.context) val actual = Tp2Adapter.toRawEvents(payload) actual must beSuccessful( - NonEmptyList(RawEvent(Snowplow.Tp2, Map("aid" -> "tp2", "e" -> "se"), None, Shared.source, Shared.context))) + NonEmptyList( + RawEvent( + Snowplow.Tp2, + Map("aid" -> "tp2", "e" -> "se"), + None, + Shared.source, + Shared.context))) } def e4 = { val body = toSelfDescJson("""[{"tv":"ios-0.1.0","p":"mob","e":"se"}]""", "payload_data") val payload = - CollectorPayload(Snowplow.Tp2, Nil, ApplicationJsonWithCharset.some, body.some, Shared.source, Shared.context) + CollectorPayload( + Snowplow.Tp2, + Nil, + ApplicationJsonWithCharset.some, + body.some, + Shared.source, + Shared.context) val actual = Tp2Adapter.toRawEvents(payload) actual must beSuccessful( NonEmptyList( - RawEvent(Snowplow.Tp2, - Map("tv" -> "ios-0.1.0", "p" -> "mob", "e" -> "se"), - ApplicationJsonWithCharset.some, - Shared.source, - Shared.context))) + RawEvent( + Snowplow.Tp2, + Map("tv" -> "ios-0.1.0", "p" -> "mob", "e" -> "se"), + ApplicationJsonWithCharset.some, + Shared.source, + Shared.context))) } def e5 = { - val body = toSelfDescJson("""[{"tv":"1","p":"1","e":"1"},{"tv":"2","p":"2","e":"2"},{"tv":"3","p":"3","e":"3"}]""", - "payload_data") - val payload = CollectorPayload(Snowplow.Tp2, - toNameValuePairs("tv" -> "0", "nuid" -> "123"), - ApplicationJsonWithCapitalCharset.some, - body.some, - Shared.source, - Shared.context) + val body = toSelfDescJson( + """[{"tv":"1","p":"1","e":"1"},{"tv":"2","p":"2","e":"2"},{"tv":"3","p":"3","e":"3"}]""", + "payload_data") + val payload = CollectorPayload( + Snowplow.Tp2, + toNameValuePairs("tv" -> "0", "nuid" -> "123"), + ApplicationJsonWithCapitalCharset.some, + body.some, + Shared.source, + Shared.context) val actual = Tp2Adapter.toRawEvents(payload) val rawEvent: RawEventParameters => RawEvent = params => - RawEvent(Snowplow.Tp2, params, ApplicationJsonWithCapitalCharset.some, Shared.source, Shared.context) + RawEvent( + Snowplow.Tp2, + params, + ApplicationJsonWithCapitalCharset.some, + Shared.source, + Shared.context) actual must beSuccessful( NonEmptyList( rawEvent(Map("tv" -> "0", "p" -> "1", "e" -> "1", "nuid" -> "123")), @@ -135,46 +169,55 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM def e6 = { val body = toSelfDescJson("""[{"tv":"ios-0.1.0","p":"mob","e":"se"}]""", "payload_data") - val payload = CollectorPayload(Snowplow.Tp2, - Nil, - ApplicationJsonWithCapitalCharset.some, - body.some, - Shared.source, - Shared.context) + val payload = CollectorPayload( + Snowplow.Tp2, + Nil, + ApplicationJsonWithCapitalCharset.some, + body.some, + Shared.source, + Shared.context) val actual = Tp2Adapter.toRawEvents(payload) actual must beSuccessful( NonEmptyList( - RawEvent(Snowplow.Tp2, - Map("tv" -> "ios-0.1.0", "p" -> "mob", "e" -> "se"), - ApplicationJsonWithCapitalCharset.some, - Shared.source, - Shared.context))) + RawEvent( + Snowplow.Tp2, + Map("tv" -> "ios-0.1.0", "p" -> "mob", "e" -> "se"), + ApplicationJsonWithCapitalCharset.some, + Shared.source, + Shared.context))) } def e7 = - "SPEC NAME" || "IN QUERYSTRING" | "IN CONTENT TYPE" | "IN BODY" | "EXP. FAILURE" | - "Invalid content type" !! Nil ! "text/plain".some ! "body".some ! "Content type of text/plain provided, expected one of: application/json, application/json; charset=utf-8" | - "Neither querystring nor body populated" !! Nil ! None ! None ! "Request body and querystring parameters empty, expected at least one populated" | - "Body populated but content type missing" !! Nil ! None ! "body".some ! "Request body provided but content type empty, expected one of: application/json, application/json; charset=utf-8" | + "SPEC NAME" || "IN QUERYSTRING" | "IN CONTENT TYPE" | "IN BODY" | "EXP. FAILURE" | + "Invalid content type" !! Nil ! "text/plain".some ! "body".some ! "Content type of text/plain provided, expected one of: application/json, application/json; charset=utf-8, application/json; charset=UTF-8" | + "Neither querystring nor body populated" !! Nil ! None ! None ! "Request body and querystring parameters empty, expected at least one populated" | + "Body populated but content type missing" !! Nil ! None ! "body".some ! "Request body provided but content type empty, expected one of: application/json, application/json; charset=utf-8, application/json; charset=UTF-8" | "Content type populated but body missing" !! toNameValuePairs("a" -> "b") ! ApplicationJsonWithCharset.some ! None ! "Content type of application/json; charset=utf-8 provided but request body empty" | - "Body is not a JSON" !! toNameValuePairs("a" -> "b") ! ApplicationJson.some ! "body".some ! "Field [Body]: invalid JSON [body] with parsing error: Unrecognized token 'body': was expecting ('true', 'false' or 'null') at [Source: java.io.StringReader@xxxxxx; line: 1, column: 9]" |> { + "Body is not a JSON" !! toNameValuePairs("a" -> "b") ! ApplicationJson.some ! "body".some ! "Field [Body]: invalid JSON [body] with parsing error: expected json value got 'body' (line 1, column 1)" |> { (_, querystring, contentType, body, expected) => { - val payload = CollectorPayload(Snowplow.Tp2, querystring, contentType, body, Shared.source, Shared.context) - val actual = Tp2Adapter.toRawEvents(payload) + val payload = CollectorPayload( + Snowplow.Tp2, + querystring, + contentType, + body, + Shared.source, + Shared.context) + val actual = Tp2Adapter.toRawEvents(payload) actual must beFailing(NonEmptyList(expected)) } } def e8 = { - val payload = CollectorPayload(Snowplow.Tp2, - toNameValuePairs("aid" -> "test"), - ApplicationJson.some, - """{"not":"self-desc"}""".some, - Shared.source, - Shared.context) + val payload = CollectorPayload( + Snowplow.Tp2, + toNameValuePairs("aid" -> "test"), + ApplicationJson.some, + """{"not":"self-desc"}""".some, + Shared.source, + Shared.context) val actual = Tp2Adapter.toRawEvents(payload) actual must beFailing( NonEmptyList( @@ -199,9 +242,15 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM } def e9 = { - val body = toSelfDescJson("""{"longitude":20.1234}""", "geolocation_context") - val payload = CollectorPayload(Snowplow.Tp2, Nil, ApplicationJson.some, body.some, Shared.source, Shared.context) - val actual = Tp2Adapter.toRawEvents(payload) + val body = toSelfDescJson("""{"longitude":20.1234}""", "geolocation_context") + val payload = CollectorPayload( + Snowplow.Tp2, + Nil, + ApplicationJson.some, + body.some, + Shared.source, + Shared.context) + val actual = Tp2Adapter.toRawEvents(payload) actual must beFailing( NonEmptyList( """error: Verifying schema as iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-* failed: found iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0 @@ -210,7 +259,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM } def e10 = - "SPEC NAME" || "IN JSON DATA" | "EXP. FAILURES" | + "SPEC NAME" || "IN JSON DATA" | "EXP. FAILURES" | "JSON object instead of array" !! "{}" ! NonEmptyList( """error: instance type (object) does not match any allowed primitive type (allowed: ["array"]) level: "error" @@ -254,7 +303,13 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM val body = toSelfDescJson(json, "payload_data") val payload = - CollectorPayload(Snowplow.Tp2, Nil, ApplicationJson.some, body.some, Shared.source, Shared.context) + CollectorPayload( + Snowplow.Tp2, + Nil, + ApplicationJson.some, + body.some, + Shared.source, + Shared.context) val actual = Tp2Adapter.toRawEvents(payload) actual must beFailing(expected) @@ -274,11 +329,11 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM NonEmptyList(RawEvent( Snowplow.Tp2, Map( - "e" -> "ue", - "tv" -> "r-tp2", + "e" -> "ue", + "tv" -> "r-tp2", "ue_pr" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}}}""", - "p" -> "web", - "cx" -> "dGVzdHRlc3R0ZXN0" + "p" -> "web", + "cx" -> "dGVzdHRlc3R0ZXN0" ), None, Shared.source, @@ -299,11 +354,11 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM NonEmptyList(RawEvent( Snowplow.Tp2, Map( - "e" -> "se", + "e" -> "se", "aid" -> "ads", - "tv" -> "r-tp2", - "co" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}}]}""", - "p" -> "web" + "tv" -> "r-tp2", + "co" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}}]}""", + "p" -> "web" ), None, Shared.source, @@ -314,7 +369,11 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM def e13 = { val payload = CollectorPayload( Snowplow.Tp2, - toNameValuePairs("u" -> "https://github.com/snowplow/snowplow", "e" -> "se", "aid" -> "ads", "co" -> ""), + toNameValuePairs( + "u" -> "https://github.com/snowplow/snowplow", + "e" -> "se", + "aid" -> "ads", + "co" -> ""), None, None, Shared.source, @@ -324,11 +383,11 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM NonEmptyList(RawEvent( Snowplow.Tp2, Map( - "e" -> "se", + "e" -> "se", "aid" -> "ads", - "tv" -> "r-tp2", - "co" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}}]}""", - "p" -> "web" + "tv" -> "r-tp2", + "co" -> """{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}}]}""", + "p" -> "web" ), None, Shared.source, @@ -340,8 +399,8 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM val payload = CollectorPayload( Snowplow.Tp2, toNameValuePairs( - "u" -> "https://github.com/snowplow/snowplow", - "e" -> "se", + "u" -> "https://github.com/snowplow/snowplow", + "e" -> "se", "co" -> """{"data":[{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}""" ), None, @@ -354,10 +413,10 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM NonEmptyList(RawEvent( Snowplow.Tp2, Map( - "e" -> "se", + "e" -> "se", "tv" -> "r-tp2", - "co" -> """{"data":[{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"},{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}""", - "p" -> "web" + "co" -> """{"data":[{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}},{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}""", + "p" -> "web" ), None, Shared.source, @@ -385,10 +444,10 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM NonEmptyList(RawEvent( Snowplow.Tp2, Map( - "e" -> "se", + "e" -> "se", "tv" -> "r-tp2", "cx" -> CU.encodeBase64Url( - """{"data":[{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"},{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}"""), + """{"data":[{"schema":"iglu:com.snowplowanalytics.snowplow/uri_redirect/jsonschema/1-0-0","data":{"uri":"https://github.com/snowplow/snowplow"}},{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}"""), "p" -> "web" ), None, @@ -399,28 +458,38 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM def e16 = { val payload = CollectorPayload(Snowplow.Tp2, Nil, None, None, Shared.source, Shared.context) - val actual = RedirectAdapter.toRawEvents(payload) + val actual = RedirectAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList("Querystring is empty: cannot be a valid URI redirect")) } def e17 = { val payload = - CollectorPayload(Snowplow.Tp2, toNameValuePairs("aid" -> "test"), None, None, Shared.source, Shared.context) + CollectorPayload( + Snowplow.Tp2, + toNameValuePairs("aid" -> "test"), + None, + None, + Shared.source, + Shared.context) val actual = RedirectAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Querystring does not contain u parameter: not a valid URI redirect")) + actual must beFailing( + NonEmptyList("Querystring does not contain u parameter: not a valid URI redirect")) } def e18 = { val payload = CollectorPayload( Snowplow.Tp2, - toNameValuePairs("u" -> "https://github.com/snowplow/snowplow", "e" -> "se", "co" -> """{[-"""), + toNameValuePairs( + "u" -> "https://github.com/snowplow/snowplow", + "e" -> "se", + "co" -> """{[-"""), None, None, Shared.source, Shared.context) val actual = RedirectAdapter.toRawEvents(payload) actual must beFailing(NonEmptyList( - """Field [co|cx]: invalid JSON [{[-] with parsing error: Unexpected character ('[' (code 91)): was expecting double-quote to start field name at [Source: (String)"{[-"; line: 1, column: 3]""")) + """Field [co|cx]: invalid JSON [{[-] with parsing error: expected " got '[-' (line 1, column 2)""")) } def e19 = { @@ -432,7 +501,8 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidationM Shared.source, Shared.context) val actual = RedirectAdapter.toRawEvents(payload) - actual must beFailing(NonEmptyList("Field [co|cx]: invalid JSON [] with parsing error: mapping resulted in null")) + actual must beFailing( + NonEmptyList("Field [co|cx]: invalid JSON [] with parsing error: exhausted input")) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala index 1d4840bf3..37924fbfe 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala @@ -37,14 +37,14 @@ class EnrichmentRegistrySpec extends Specification with DataTables with Validati def makeFiles(files: String*): List[(URI, String)] = files.toList.map(f => (new URI(s"http://foobar.com/$f"), f)) - val nofiles = NoFileEnrichment() + val nofiles = NoFileEnrichment() val enrichment1 = FileEnrichment(files1) val enrichment2 = FileEnrichment(files2) def e1 = - "SPEC NAME" || "ENRICHMENTS" | "EXPECTED FILES" | - "none with files" !! enrichments(nofiles, nofiles, nofiles) ! List.empty | - "one with files" !! enrichments(nofiles, enrichment1, nofiles) ! files1 | + "SPEC NAME" || "ENRICHMENTS" | "EXPECTED FILES" | + "none with files" !! enrichments(nofiles, nofiles, nofiles) ! List.empty | + "one with files" !! enrichments(nofiles, enrichment1, nofiles) ! files1 | "multiple with files" !! enrichments(enrichment1, nofiles, enrichment2) ! files1 ++ files2 |> { (_, enrichments, expectedFiles) => { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentTest.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentTest.scala index a61d2cda3..379916308 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentTest.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentTest.scala @@ -34,32 +34,42 @@ class SchemaEnrichmentTest extends Specification with DataTables with Validation """ def e1 = - "SPEC NAME" || "EVENT" | "EXPECTED SCHEMA" | - "page view" !! event("page_view") ! SchemaKey("com.snowplowanalytics.snowplow", - "page_view", - "jsonschema", - "1-0-0") | - "ping ping" !! event("page_ping") ! SchemaKey("com.snowplowanalytics.snowplow", - "page_ping", - "jsonschema", - "1-0-0") | - "transaction" !! event("transaction") ! SchemaKey("com.snowplowanalytics.snowplow", - "transaction", - "jsonschema", - "1-0-0") | - "transaction item" !! event("transaction_item") ! SchemaKey("com.snowplowanalytics.snowplow", - "transaction_item", - "jsonschema", - "1-0-0") | - "struct event" !! event("struct") ! SchemaKey("com.google.analytics", "event", "jsonschema", "1-0-0") | - "invalid unstruct event" !! unstructEvent(invalidPayload) ! SchemaKey("com.snowplowanalytics.snowplow-website", - "signup_form_submitted", - "jsonschema", - "1-0-0") | - "unstruct event" !! unstructEvent(signupFormSubmitted) ! SchemaKey("com.snowplowanalytics.snowplow-website", - "signup_form_submitted", - "jsonschema", - "1-0-0") |> { (_, event, expected) => + "SPEC NAME" || "EVENT" | "EXPECTED SCHEMA" | + "page view" !! event("page_view") ! SchemaKey( + "com.snowplowanalytics.snowplow", + "page_view", + "jsonschema", + "1-0-0") | + "ping ping" !! event("page_ping") ! SchemaKey( + "com.snowplowanalytics.snowplow", + "page_ping", + "jsonschema", + "1-0-0") | + "transaction" !! event("transaction") ! SchemaKey( + "com.snowplowanalytics.snowplow", + "transaction", + "jsonschema", + "1-0-0") | + "transaction item" !! event("transaction_item") ! SchemaKey( + "com.snowplowanalytics.snowplow", + "transaction_item", + "jsonschema", + "1-0-0") | + "struct event" !! event("struct") ! SchemaKey( + "com.google.analytics", + "event", + "jsonschema", + "1-0-0") | + "invalid unstruct event" !! unstructEvent(invalidPayload) ! SchemaKey( + "com.snowplowanalytics.snowplow-website", + "signup_form_submitted", + "jsonschema", + "1-0-0") | + "unstruct event" !! unstructEvent(signupFormSubmitted) ! SchemaKey( + "com.snowplowanalytics.snowplow-website", + "signup_form_submitted", + "jsonschema", + "1-0-0") |> { (_, event, expected) => { val schema = SchemaEnrichment.extractSchema(event) schema must beSuccessful(expected) @@ -72,11 +82,11 @@ class SchemaEnrichmentTest extends Specification with DataTables with Validation """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow-website/signup_form_submitted/jsonschema","data":{"name":"Χαριτίνη NEW Unicode test","email":"alex+test@snowplowanalytics.com","company":"SP","eventsPerMonth":"< 1 million","serviceType":"unsure"}}}""" def e2 = - "SPEC NAME" || "EVENT" | + "SPEC NAME" || "EVENT" | "unknown event" !! event("unknown") | "missing event" !! event(null) | - "not schemed" !! unstructEvent(nonSchemedPayload) | - "invalid key" !! unstructEvent(invalidKeyPayload) |> { (_, event) => + "not schemed" !! unstructEvent(nonSchemedPayload) | + "invalid key" !! unstructEvent(invalidKeyPayload) |> { (_, event) => { val schema = SchemaEnrichment.extractSchema(event) schema must beFailing diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala index 8c0d705f8..05570889b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala @@ -30,16 +30,17 @@ class ExtractViewDimensionsSpec extends Specification with DataTables { s2"Extracting screen dimensions (viewports, screen resolution etc) with extractViewDimensions should work $e1" def e1 = - "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | - "valid desktop" !! "1200x800" ! (1200, 800).success | - "valid mobile" !! "76x128" ! (76, 128).success | - "invalid empty" !! "" ! err("").fail | - "invalid null" !! null ! err(null).fail | - "invalid hex" !! "76xEE" ! err("76xEE").fail | - "invalid negative" !! "1200x-17" ! err("1200x-17").fail | - "Arabic number" !! "٤٥٦٧x680" ! err("٤٥٦٧x680").fail | - "number > int #1" !! "760x3389336768" ! err2("760x3389336768").fail | - "number > int #2" !! "9989336768x1200" ! err2("9989336768x1200").fail |> { (_, input, expected) => - ClientEnrichments.extractViewDimensions(FieldName, input) must_== expected + "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | + "valid desktop" !! "1200x800" ! (1200, 800).success | + "valid mobile" !! "76x128" ! (76, 128).success | + "invalid empty" !! "" ! err("").fail | + "invalid null" !! null ! err(null).fail | + "invalid hex" !! "76xEE" ! err("76xEE").fail | + "invalid negative" !! "1200x-17" ! err("1200x-17").fail | + "Arabic number" !! "٤٥٦٧x680" ! err("٤٥٦٧x680").fail | + "number > int #1" !! "760x3389336768" ! err2("760x3389336768").fail | + "number > int #2" !! "9989336768x1200" ! err2("9989336768x1200").fail |> { + (_, input, expected) => + ClientEnrichments.extractViewDimensions(FieldName, input) must_== expected } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala index da46a5ea8..65c7f296f 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala @@ -30,34 +30,35 @@ class ExtractEventTypeSpec extends Specification with DataTables with Validation """ val FieldName = "e" - def err: (String) => String = input => "Field [%s]: [%s] is not a recognised event code".format(FieldName, input) + def err: (String) => String = + input => "Field [%s]: [%s] is not a recognised event code".format(FieldName, input) def e1 = - "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | - "transaction" !! "tr" ! "transaction" | - "transaction item" !! "ti" ! "transaction_item" | - "page view" !! "pv" ! "page_view" | - "page ping" !! "pp" ! "page_ping" | - "unstructured event" !! "ue" ! "unstruct" | - "structured event" !! "se" ! "struct" | - "structured event (legacy)" !! "ev" ! "struct" | - "ad impression (legacy)" !! "ad" ! "ad_impression" |> { (_, input, expected) => + "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | + "transaction" !! "tr" ! "transaction" | + "transaction item" !! "ti" ! "transaction_item" | + "page view" !! "pv" ! "page_view" | + "page ping" !! "pp" ! "page_ping" | + "unstructured event" !! "ue" ! "unstruct" | + "structured event" !! "se" ! "struct" | + "structured event (legacy)" !! "ev" ! "struct" | + "ad impression (legacy)" !! "ad" ! "ad_impression" |> { (_, input, expected) => EventEnrichments.extractEventType(FieldName, input) must beSuccessful(expected) } def e2 = - "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | - "null" !! null ! err("null") | - "empty string" !! "" ! err("") | - "unrecognized #1" !! "e" ! err("e") | - "unrecognized #2" !! "evnt" ! err("evnt") |> { (_, input, expected) => + "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | + "null" !! null ! err("null") | + "empty string" !! "" ! err("") | + "unrecognized #1" !! "e" ! err("e") | + "unrecognized #2" !! "evnt" ! err("evnt") |> { (_, input, expected) => EventEnrichments.extractEventType(FieldName, input) must beFailing(expected) } val SeventiesTstamp = Some(new DateTime(0, DateTimeZone.UTC)) - val BCTstamp = SeventiesTstamp.map(_.minusYears(2000)) - val FarAwayTstamp = SeventiesTstamp.map(_.plusYears(10000)) - def e3 = + val BCTstamp = SeventiesTstamp.map(_.minusYears(2000)) + val FarAwayTstamp = SeventiesTstamp.map(_.plusYears(10000)) + def e3 = // format: off "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | "None" !! None ! "No collector_tstamp set".fail | @@ -85,16 +86,19 @@ class DerivedTimestampSpec extends Specification with DataTables with Validation "getDerivedTimestamp should correctly calculate the derived timestamp " ! e1 ^ end def e1 = - "SPEC NAME" || "DVCE_CREATED_TSTAMP" | "DVCE_SENT_TSTAMP" | "COLLECTOR_TSTAMP" | "TRUE_TSTAMP" | "EXPECTED DERIVED_TSTAMP" | - "No dvce_sent_tstamp" !! "2014-04-29 12:00:54.555" ! null ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:54.000" | - "No dvce_created_tstamp" !! null ! null ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:54.000" | - "No collector_tstamp" !! null ! null ! null ! null ! null | - "dvce_sent_tstamp before dvce_created_tstamp" !! "2014-04-29 09:00:54.001" ! "2014-04-29 09:00:54.000" ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:54.000" | - "dvce_sent_tstamp after dvce_created_tstamp" !! "2014-04-29 09:00:54.000" ! "2014-04-29 09:00:54.001" ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:53.999" | - "true_tstamp override" !! "2014-04-29 09:00:54.001" ! "2014-04-29 09:00:54.000" ! "2014-04-29 09:00:54.000" ! "2000-01-01 00:00:00.000" ! "2000-01-01 00:00:00.000" |> { + "SPEC NAME" || "DVCE_CREATED_TSTAMP" | "DVCE_SENT_TSTAMP" | "COLLECTOR_TSTAMP" | "TRUE_TSTAMP" | "EXPECTED DERIVED_TSTAMP" | + "No dvce_sent_tstamp" !! "2014-04-29 12:00:54.555" ! null ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:54.000" | + "No dvce_created_tstamp" !! null ! null ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:54.000" | + "No collector_tstamp" !! null ! null ! null ! null ! null | + "dvce_sent_tstamp before dvce_created_tstamp" !! "2014-04-29 09:00:54.001" ! "2014-04-29 09:00:54.000" ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:54.000" | + "dvce_sent_tstamp after dvce_created_tstamp" !! "2014-04-29 09:00:54.000" ! "2014-04-29 09:00:54.001" ! "2014-04-29 09:00:54.000" ! null ! "2014-04-29 09:00:53.999" | + "true_tstamp override" !! "2014-04-29 09:00:54.001" ! "2014-04-29 09:00:54.000" ! "2014-04-29 09:00:54.000" ! "2000-01-01 00:00:00.000" ! "2000-01-01 00:00:00.000" |> { (_, created, sent, collected, truth, expected) => - EventEnrichments.getDerivedTimestamp(Option(sent), Option(created), Option(collected), Option(truth)) must beSuccessful( - Option(expected)) + EventEnrichments.getDerivedTimestamp( + Option(sent), + Option(created), + Option(collected), + Option(truth)) must beSuccessful(Option(expected)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala index d72bd6722..3f6462e0d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala @@ -12,29 +12,25 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments +import io.circe.literal._ import org.specs2.mutable.{Specification => MutSpecification} import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import scalaz._ import Scalaz._ -import org.json4s.JsonDSL._ class EtlVersionSpec extends MutSpecification { - "The ETL version" should { "be successfully returned using an x.y.z format" in { val anyString = "spark-x.x.x" - MiscEnrichments.etlVersion(anyString) must beMatching(s"${anyString}-common-\\d+\\.\\d+\\.\\d+(-\\w+)?".r) + MiscEnrichments.etlVersion(anyString) must beMatching( + s"${anyString}-common-\\d+\\.\\d+\\.\\d+(-\\w+)?".r) } } } -/** - * Tests the extractPlatform function. - * Uses DataTables. - */ +/** Tests the extractPlatform function. Uses DataTables. */ class ExtractPlatformSpec extends Specification with DataTables { - val FieldName = "p" def err: (String) => String = input => "Field [%s]: [%s] is not a supported tracking platform".format(FieldName, input) @@ -42,18 +38,18 @@ class ExtractPlatformSpec extends Specification with DataTables { def is = s2"Extracting platforms with extractPlatform should work $e1" def e1 = - "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | - "valid web" !! "web" ! "web".success | - "valid mobile/tablet" !! "mob" ! "mob".success | - "valid desktop/laptop/netbook" !! "pc" ! "pc".success | - "valid server-side app" !! "srv" ! "srv".success | - "valid general app" !! "app" ! "app".success | - "valid connected TV" !! "tv" ! "tv".success | - "valid games console" !! "cnsl" ! "cnsl".success | - "valid iot (internet of things)" !! "iot" ! "iot".success | - "invalid empty" !! "" ! err("").fail | - "invalid null" !! null ! err(null).fail | - "invalid platform" !! "ma" ! err("ma").fail |> { (_, input, expected) => + "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | + "valid web" !! "web" ! "web".success | + "valid mobile/tablet" !! "mob" ! "mob".success | + "valid desktop/laptop/netbook" !! "pc" ! "pc".success | + "valid server-side app" !! "srv" ! "srv".success | + "valid general app" !! "app" ! "app".success | + "valid connected TV" !! "tv" ! "tv".success | + "valid games console" !! "cnsl" ! "cnsl".success | + "valid iot (internet of things)" !! "iot" ! "iot".success | + "invalid empty" !! "" ! err("").fail | + "invalid null" !! null ! err(null).fail | + "invalid platform" !! "ma" ! err("ma").fail |> { (_, input, expected) => MiscEnrichments.extractPlatform(FieldName, input) must_== expected } } @@ -65,14 +61,14 @@ class ExtractIpSpec extends Specification with DataTables { val nullString: String = null def e1 = - "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | - "single ip" !! "127.0.0.1" ! "127.0.0.1".success | - "ips ', '-separated" !! "127.0.0.1, 127.0.0.2" ! "127.0.0.1".success | - "ips ','-separated" !! "127.0.0.1,127.0.0.2" ! "127.0.0.1".success | - "ips separated out of the spec" !! "1.0.0.1!1.0.0.2" ! "1.0.0.1!1.0.0.2".success | + "SPEC NAME" || "INPUT VAL" | "EXPECTED OUTPUT" | + "single ip" !! "127.0.0.1" ! "127.0.0.1".success | + "ips ', '-separated" !! "127.0.0.1, 127.0.0.2" ! "127.0.0.1".success | + "ips ','-separated" !! "127.0.0.1,127.0.0.2" ! "127.0.0.1".success | + "ips separated out of the spec" !! "1.0.0.1!1.0.0.2" ! "1.0.0.1!1.0.0.2".success | // ConversionUtils.makeTsvSafe returns null for empty string - "empty" !! "" ! Success(null) | - "null" !! null ! Success(null) |> { (_, input, expected) => + "empty" !! "" ! Success(null) | + "null" !! null ! Success(null) |> { (_, input, expected) => MiscEnrichments.extractIp("ip", input) must_== expected } @@ -99,14 +95,24 @@ class FormatDerivedContextsSpec extends MutSpecification { "convert a list of JObjects to a self-describing contexts JSON" in { val derivedContextsList = List( - (("schema" -> "iglu:com.acme/user/jsonschema/1-0-0") ~ - ("data" -> - ("type" -> "tester") ~ - ("name" -> "bethany"))), - (("schema" -> "iglu:com.acme/design/jsonschema/1-0-0") ~ - ("data" -> - ("color" -> "red") ~ - ("fontSize" -> 14))) + json""" + { + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "data": { + "type": "tester", + "name": "bethany" + } + } + """, + json""" + { + "schema": "iglu:com.acme/design/jsonschema/1-0-0", + "data": { + "color": "red", + "fontSize": 14 + } + } + """ ) val expected = """ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala index 6246c6717..1dd0af291 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala @@ -26,34 +26,34 @@ class CampaignAttributionEnrichmentSpec extends Specification with ValidationMat """ val google_uri = Map( - "utm_source" -> "GoogleSearch", - "utm_medium" -> "cpc", - "utm_term" -> "native american tarot deck", - "utm_content" -> "39254295088", + "utm_source" -> "GoogleSearch", + "utm_medium" -> "cpc", + "utm_term" -> "native american tarot deck", + "utm_content" -> "39254295088", "utm_campaign" -> "uk-tarot--native-american" ) val omniture_uri = Map("cid" -> "uk-tarot--native-american") val heterogeneous_uri = Map( - "utm_source" -> "GoogleSearch", - "source" -> "bad_source", - "utm_medium" -> "cpc", - "legacy_term" -> "bad_term", - "utm_term" -> "native american tarot deck", + "utm_source" -> "GoogleSearch", + "source" -> "bad_source", + "utm_medium" -> "cpc", + "legacy_term" -> "bad_term", + "utm_term" -> "native american tarot deck", "legacy_campaign" -> "bad_campaign", - "cid" -> "uk-tarot--native-american" + "cid" -> "uk-tarot--native-american" ) val clickid_uri = Map( - "utm_source" -> "GoogleSearch", - "source" -> "bad_source", - "utm_medium" -> "cpc", - "legacy_term" -> "bad_term", - "utm_term" -> "native american tarot deck", + "utm_source" -> "GoogleSearch", + "source" -> "bad_source", + "utm_medium" -> "cpc", + "legacy_term" -> "bad_term", + "utm_term" -> "native american tarot deck", "legacy_campaign" -> "bad_campaign", - "cid" -> "uk-tarot--native-american", - "msclkid" -> "500" + "cid" -> "uk-tarot--native-american", + "msclkid" -> "500" ) def e1 = { @@ -81,13 +81,14 @@ class CampaignAttributionEnrichmentSpec extends Specification with ValidationMat ) config.extractMarketingFields(google_uri) must beSuccessful( - MarketingCampaign(Some("cpc"), - Some("GoogleSearch"), - Some("native american tarot deck"), - Some("39254295088"), - Some("uk-tarot--native-american"), - None, - None)) + MarketingCampaign( + Some("cpc"), + Some("GoogleSearch"), + Some("native american tarot deck"), + Some("39254295088"), + Some("uk-tarot--native-american"), + None, + None)) } def e3 = { @@ -115,13 +116,14 @@ class CampaignAttributionEnrichmentSpec extends Specification with ValidationMat ) config.extractMarketingFields(heterogeneous_uri) must beSuccessful( - MarketingCampaign(Some("cpc"), - Some("GoogleSearch"), - Some("native american tarot deck"), - None, - Some("uk-tarot--native-american"), - None, - None)) + MarketingCampaign( + Some("cpc"), + Some("GoogleSearch"), + Some("native american tarot deck"), + None, + Some("uk-tarot--native-american"), + None, + None)) } def e5 = { @@ -132,20 +134,21 @@ class CampaignAttributionEnrichmentSpec extends Specification with ValidationMat List("utm_content"), List("utm_campaign", "cid", "legacy_campaign"), List( - "gclid" -> "Google", + "gclid" -> "Google", "msclkid" -> "Microsoft", - "dclid" -> "DoubleClick" + "dclid" -> "DoubleClick" ) ) config.extractMarketingFields(clickid_uri) must beSuccessful( - MarketingCampaign(Some("cpc"), - Some("GoogleSearch"), - Some("native american tarot deck"), - None, - Some("uk-tarot--native-american"), - Some("500"), - Some("Microsoft"))) + MarketingCampaign( + Some("cpc"), + Some("GoogleSearch"), + Some("native american tarot deck"), + None, + Some("uk-tarot--native-american"), + Some("500"), + Some("Microsoft"))) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala index a9df50f84..b5d5455af 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala @@ -11,10 +11,9 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry +import io.circe.literal._ import org.specs2.Specification import org.specs2.scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ class CookieExtractorEnrichmentSpec extends Specification with ValidationMatchers { def is = s2""" @@ -31,29 +30,28 @@ class CookieExtractorEnrichmentSpec extends Specification with ValidationMatcher } def e2 = { - val actual = CookieExtractorEnrichment(List("cookieKey1")).extract(List("Cookie: not-interesting-cookie=1234;")) - + val actual = CookieExtractorEnrichment(List("cookieKey1")) + .extract(List("Cookie: not-interesting-cookie=1234;")) actual must_== Nil } def e3 = { - val cookies = List("ck1", "=cv2", "ck3=", "ck4=cv4", "ck5=\"cv5\"") + val cookies = List("ck1", "=cv2", "ck3=", "ck4=cv4", "ck5=\"cv5\"") val cookieKeys = List("ck1", "", "ck3", "ck4", "ck5") val expected = List( - """{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck1","value":null}}""", - """{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"","value":"cv2"}}""", - """{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck3","value":""}}""", - """{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck4","value":"cv4"}}""", - """{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck5","value":"cv5"}}""" + json"""{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck1","value":null}}""", + json"""{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"","value":"cv2"}}""", + json"""{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck3","value":""}}""", + json"""{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck4","value":"cv4"}}""", + json"""{"schema":"iglu:org.ietf/http_cookie/jsonschema/1-0-0","data":{"name":"ck5","value":"cv5"}}""" ) - val actual = CookieExtractorEnrichment(cookieKeys).extract(List("Cookie: " + cookies.mkString(";"))) + val actual = CookieExtractorEnrichment(cookieKeys) + .extract(List("Cookie: " + cookies.mkString(";"))) actual must beLike { - case cookies @ _ :: _ :: _ :: _ :: _ :: Nil => { - cookies.map(c => compact(render(c))) must_== expected.map(e => compact(render(parse(e)))) - } + case cookies @ _ :: _ :: _ :: _ :: _ :: Nil => cookies must_== expected } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala index 7014f21fb..4818144a4 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala @@ -23,9 +23,7 @@ object CurrencyConversionEnrichmentSpec { val OerApiKey = "OER_KEY" } -/** - * Tests the convertCurrencies function - */ +/** Tests the convertCurrencies function */ import CurrencyConversionEnrichmentSpec._ class CurrencyConversionEnrichmentSpec extends Specification with DataTables { def is = @@ -38,8 +36,8 @@ class CurrencyConversionEnrichmentSpec extends Specification with DataTables { lazy val validAppKey = sys.env .get(OerApiKey) - .getOrElse( - throw new IllegalStateException(s"No ${OerApiKey} environment variable found, test should have been skipped")) + .getOrElse(throw new IllegalStateException( + s"No ${OerApiKey} environment variable found, test should have been skipped")) val trCurrencyMissing = Failure( NonEmptyList( "Open Exchange Rates error, message: Currency [] is not supported by Joda money Currency not found in the API, invalid currency ", @@ -59,57 +57,84 @@ class CurrencyConversionEnrichmentSpec extends Specification with DataTables { val coTstamp: DateTime = new DateTime(2011, 3, 13, 0, 0) def e1 = - "SPEC NAME" || "TRANSACTION CURRENCY" | "API KEY" | "TOTAL AMOUNT" | "TOTAL TAX" | "SHIPPING" | "TRANSACTION ITEM CURRENCY" | "TRANSACTION ITEM PRICE" | "DATETIME" | "CONVERTED TUPLE" | - "Invalid transaction currency" !! Some("RUP") ! validAppKey ! Some(11.00) ! Some(1.17) ! Some(0.00) ! None ! Some( - 17.99) ! Some(coTstamp) ! currencyInvalidRup | - "Invalid transaction item currency" !! None ! validAppKey ! Some(12.00) ! Some(0.7) ! Some(0.00) ! Some("HUL") ! Some( - 1.99) ! Some(coTstamp) ! currencyInvalidHul | - "Invalid OER API key" !! None ! "8A8A8A8A8A8A8A8A8A8A8A8AA8A8A8A8" ! Some(13.00) ! Some(3.67) ! Some(0.00) ! Some( - "GBP") ! Some(2.99) ! Some(coTstamp) ! invalidAppKeyFailure |> { - (_, trCurrency, apiKey, trAmountTotal, trAmountTax, trAmountShipping, tiCurrency, tiPrice, dateTime, expected) => - CurrencyConversionEnrichment(DeveloperAccount, apiKey, "EUR", "EOD_PRIOR").convertCurrencies( - trCurrency, - trAmountTotal, - trAmountTax, - trAmountShipping, - tiCurrency, - tiPrice, - dateTime) must_== expected + "SPEC NAME" || "TRANSACTION CURRENCY" | "API KEY" | "TOTAL AMOUNT" | "TOTAL TAX" | "SHIPPING" | "TRANSACTION ITEM CURRENCY" | "TRANSACTION ITEM PRICE" | "DATETIME" | "CONVERTED TUPLE" | + "Invalid transaction currency" !! Some("RUP") ! validAppKey ! Some(11.00) ! Some(1.17) ! Some( + 0.00) ! None ! Some(17.99) ! Some(coTstamp) ! currencyInvalidRup | + "Invalid transaction item currency" !! None ! validAppKey ! Some(12.00) ! Some(0.7) ! Some( + 0.00) ! Some("HUL") ! Some(1.99) ! Some(coTstamp) ! currencyInvalidHul | + "Invalid OER API key" !! None ! "8A8A8A8A8A8A8A8A8A8A8A8AA8A8A8A8" ! Some(13.00) ! Some(3.67) ! Some( + 0.00) ! Some("GBP") ! Some(2.99) ! Some(coTstamp) ! invalidAppKeyFailure |> { + ( + _, + trCurrency, + apiKey, + trAmountTotal, + trAmountTax, + trAmountShipping, + tiCurrency, + tiPrice, + dateTime, + expected) => + CurrencyConversionEnrichment(DeveloperAccount, apiKey, "EUR", "EOD_PRIOR") + .convertCurrencies( + trCurrency, + trAmountTotal, + trAmountTax, + trAmountShipping, + tiCurrency, + tiPrice, + dateTime) must_== expected } def e2 = - "SPEC NAME" || "TRANSACTION CURRENCY" | "API KEY" | "TOTAL AMOUNT" | "TOTAL TAX" | "SHIPPING" | "TRANSACTION ITEM CURRENCY" | "TRANSACTION ITEM PRICE" | "DATETIME" | "CONVERTED TUPLE" | - "All fields absent" !! None ! validAppKey ! None ! None ! None ! None ! None ! None ! Failure( - NonEmptyList("Collector timestamp missing")) | - "All fields absent except currency" !! Some("GBP") ! validAppKey ! None ! None ! None ! Some("GBP") ! None ! None ! Failure( + "SPEC NAME" || "TRANSACTION CURRENCY" | "API KEY" | "TOTAL AMOUNT" | "TOTAL TAX" | "SHIPPING" | "TRANSACTION ITEM CURRENCY" | "TRANSACTION ITEM PRICE" | "DATETIME" | "CONVERTED TUPLE" | + "All fields absent" !! None ! validAppKey ! None ! None ! None ! None ! None ! None ! Failure( NonEmptyList("Collector timestamp missing")) | + "All fields absent except currency" !! Some("GBP") ! validAppKey ! None ! None ! None ! Some( + "GBP") ! None ! None ! Failure(NonEmptyList("Collector timestamp missing")) | "No transaction currency, tax, or shipping" !! Some("GBP") ! validAppKey ! Some(11.00) ! None ! None ! None ! None ! Some( - coTstamp) ! (Some("12.75"), None, None, None).success | - "No transaction currency or total" !! Some("GBP") ! validAppKey ! None ! Some(2.67) ! Some(0.00) ! None ! None ! Some( - coTstamp) ! (None, Some("3.09"), Some("0.00"), None).success | - "No transaction currency" !! None ! validAppKey ! None ! None ! None ! Some("GBP") ! Some(12.99) ! Some(coTstamp) ! (None, - None, - None, - Some( - "15.05")).success | - "Transaction Item Null" !! Some("GBP") ! validAppKey ! Some(11.00) ! Some(2.67) ! Some(0.00) ! None ! None ! Some( - coTstamp) ! (Some("12.75"), Some("3.09"), Some("0.00"), None).success | - "Valid APP ID and API key" !! None ! validAppKey ! Some(14.00) ! Some(4.67) ! Some(0.00) ! Some("GBP") ! Some( - 10.99) ! Some(coTstamp) ! (None, None, None, Some("12.74")).success | - "Both Currency Null" !! None ! validAppKey ! Some(11.00) ! Some(2.67) ! Some(0.00) ! None ! Some(12.99) ! Some( - coTstamp) ! (None, None, None, None).success | - "Convert to the same currency" !! Some("EUR") ! validAppKey ! Some(11.00) ! Some(2.67) ! Some(0.00) ! Some("EUR") ! Some( - 12.99) ! Some(coTstamp) ! (Some("11.00"), Some("2.67"), Some("0.00"), Some("12.99")).success | - "Valid APP ID and API key" !! Some("GBP") ! validAppKey ! Some(16.00) ! Some(2.67) ! Some(0.00) ! None ! Some( - 10.00) ! Some(coTstamp) ! (Some("18.54"), Some("3.09"), Some("0.00"), None).success |> { - (_, trCurrency, apiKey, trAmountTotal, trAmountTax, trAmountShipping, tiCurrency, tiPrice, dateTime, expected) => - CurrencyConversionEnrichment(DeveloperAccount, apiKey, "EUR", "EOD_PRIOR").convertCurrencies( - trCurrency, - trAmountTotal, - trAmountTax, - trAmountShipping, - tiCurrency, - tiPrice, - dateTime) must_== expected + coTstamp) ! (Some("12.75"), None, None, None).success | + "No transaction currency or total" !! Some("GBP") ! validAppKey ! None ! Some(2.67) ! Some( + 0.00) ! None ! None ! Some(coTstamp) ! (None, Some("3.09"), Some("0.00"), None).success | + "No transaction currency" !! None ! validAppKey ! None ! None ! None ! Some("GBP") ! Some( + 12.99) ! Some(coTstamp) ! (None, None, None, Some("15.05")).success | + "Transaction Item Null" !! Some("GBP") ! validAppKey ! Some(11.00) ! Some(2.67) ! Some(0.00) ! None ! None ! Some( + coTstamp) ! (Some("12.75"), Some("3.09"), Some("0.00"), None).success | + "Valid APP ID and API key" !! None ! validAppKey ! Some(14.00) ! Some(4.67) ! Some(0.00) ! Some( + "GBP") ! Some(10.99) ! Some(coTstamp) ! (None, None, None, Some("12.74")).success | + "Both Currency Null" !! None ! validAppKey ! Some(11.00) ! Some(2.67) ! Some(0.00) ! None ! Some( + 12.99) ! Some(coTstamp) ! (None, None, None, None).success | + "Convert to the same currency" !! Some("EUR") ! validAppKey ! Some(11.00) ! Some(2.67) ! Some( + 0.00) ! Some("EUR") ! Some(12.99) ! Some(coTstamp) ! ( + Some("11.00"), + Some("2.67"), + Some("0.00"), + Some("12.99")).success | + "Valid APP ID and API key" !! Some("GBP") ! validAppKey ! Some(16.00) ! Some(2.67) ! Some( + 0.00) ! None ! Some(10.00) ! Some(coTstamp) ! ( + Some("18.54"), + Some("3.09"), + Some("0.00"), + None).success |> { + ( + _, + trCurrency, + apiKey, + trAmountTotal, + trAmountTax, + trAmountShipping, + tiCurrency, + tiPrice, + dateTime, + expected) => + CurrencyConversionEnrichment(DeveloperAccount, apiKey, "EUR", "EOD_PRIOR") + .convertCurrencies( + trCurrency, + trAmountTotal, + trAmountTax, + trAmountShipping, + tiCurrency, + tiPrice, + dateTime) must_== expected } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala index 83c349bc8..46f188fe1 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala @@ -14,10 +14,12 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.net.URI +import cats.syntax.either._ import com.snowplowanalytics.forex.oerclient.DeveloperAccount import com.snowplowanalytics.iglu.client.SchemaKey +import io.circe.literal._ +import io.circe.parser._ import org.apache.commons.codec.binary.Base64 -import org.json4s.jackson.JsonMethods.parse import org.specs2.matcher.DataTables import org.specs2.mutable.Specification import org.specs2.scalaz.ValidationMatchers @@ -26,16 +28,13 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid anon_ip enrichment JSON" should { "successfully construct an AnonIpEnrichment case class with default value for IPv6" in { - - val ipAnonJson = parse("""{ + val ipAnonJson = json"""{ "enabled": true, "parameters": { "anonOctets": 2 } - }""") - + }""" val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "anon_ip", "jsonschema", "1-0-0") - val result = AnonIpEnrichment.parse(ipAnonJson, schemaKey) result must beSuccessful(AnonIpEnrichment(AnonIPv4Octets(2), AnonIPv6Segments(2))) @@ -63,8 +62,7 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid ip_lookups enrichment JSON" should { "successfully construct a GeoIpEnrichment case class" in { - - val ipToGeoJson = parse("""{ + val ipToGeoJson = json"""{ "enabled": true, "parameters": { "geo": { @@ -76,17 +74,21 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "uri": "http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind" } } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "ip_lookups", "jsonschema", "2-0-0") - + }""" + val schemaKey = + SchemaKey("com.snowplowanalytics.snowplow", "ip_lookups", "jsonschema", "2-0-0") val expected = IpLookupsEnrichment( - Some(("geo", - new URI("http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind/GeoIP2-City.mmdb"), - "GeoIP2-City.mmdb")), - Some(("isp", - new URI("http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind/GeoIP2-ISP.mmdb"), - "GeoIP2-ISP.mmdb")), + Some(( + "geo", + new URI( + "http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind/GeoIP2-City.mmdb"), + "GeoIP2-City.mmdb")), + Some( + ( + "isp", + new URI( + "http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind/GeoIP2-ISP.mmdb"), + "GeoIP2-ISP.mmdb")), None, None, true @@ -100,8 +102,7 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid referer_parser enrichment JSON" should { "successfully construct a RefererParserEnrichment case class" in { - - val refererParserJson = parse("""{ + val refererParserJson = json"""{ "enabled": true, "parameters": { "internalDomains": [ @@ -109,13 +110,12 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "www.subdomain2.snowplowanalytics.com" ] } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "referer_parser", "jsonschema", "1-0-0") - + }""" + val schemaKey = + SchemaKey("com.snowplowanalytics.snowplow", "referer_parser", "jsonschema", "1-0-0") val expected = - RefererParserEnrichment(List("www.subdomain1.snowplowanalytics.com", "www.subdomain2.snowplowanalytics.com")) - + RefererParserEnrichment( + List("www.subdomain1.snowplowanalytics.com", "www.subdomain2.snowplowanalytics.com")) val result = RefererParserEnrichment.parse(refererParserJson, schemaKey) result must beSuccessful(expected) @@ -124,28 +124,27 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid campaign_attribution enrichment JSON" should { "successfully construct a CampaignAttributionEnrichment case class" in { - val campaignAttributionEnrichmentJson = - parse("""{ - "enabled": true, - "parameters": { - "mapping": "static", - "fields": { - "mktMedium": ["utm_medium", "medium"], - "mktSource": ["utm_source", "source"], - "mktTerm": ["utm_term"], - "mktContent": [], - "mktCampaign": ["utm _ campaign", "CID", "legacy-campaign!?-`@#$%^&*()=\\][}{/.,<>~|"], - "mktClickId": { - "customclid": "Custom", - "gclid": "Override" + parse( + """{ + "enabled": true, + "parameters": { + "mapping": "static", + "fields": { + "mktMedium": ["utm_medium", "medium"], + "mktSource": ["utm_source", "source"], + "mktTerm": ["utm_term"], + "mktContent": [], + "mktCampaign": ["utm _ campaign", "CID", "legacy-campaign!?-`@#$%^&*()=\\][}{/.,<>~|"], + "mktClickId": { + "customclid": "Custom", + "gclid": "Override" + } } } - } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "campaign_attribution", "jsonschema", "1-0-0") - + }""").toOption.get + val schemaKey = + SchemaKey("com.snowplowanalytics.snowplow", "campaign_attribution", "jsonschema", "1-0-0") val expected = CampaignAttributionEnrichment( List("utm_medium", "medium"), List("utm_source", "source"), @@ -153,30 +152,29 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D List(), List("utm _ campaign", "CID", "legacy-campaign!?-`@#$%^&*()=\\][}{/.,<>~|"), List( - "gclid" -> "Override", - "msclkid" -> "Microsoft", - "dclid" -> "DoubleClick", + "gclid" -> "Override", + "msclkid" -> "Microsoft", + "dclid" -> "DoubleClick", "customclid" -> "Custom" ) ) - val result = CampaignAttributionEnrichment.parse(campaignAttributionEnrichmentJson, schemaKey) result must beSuccessful(expected) - } } "Parsing a valid user_agent_utils_config enrichment JSON" should { "successfully construct a UserAgentUtilsEnrichment case object" in { - - val userAgentUtilsEnrichmentJson = parse("""{ + val userAgentUtilsEnrichmentJson = json"""{ "enabled": true, "parameters": { } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "user_agent_utils_config", "jsonschema", "1-0-0") - + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "user_agent_utils_config", + "jsonschema", + "1-0-0") val result = UserAgentUtilsEnrichmentConfig.parse(userAgentUtilsEnrichmentJson, schemaKey) result must beSuccessful(UserAgentUtilsEnrichment) @@ -185,27 +183,25 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid ua_parser_config enrichment JSON" should { "successfully construct a UaParserEnrichment case class" in { - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "ua_parser_config", "jsonschema", "1-0-1") - - val configWithDefaultRules = parse("""{ + val schemaKey = + SchemaKey("com.snowplowanalytics.snowplow", "ua_parser_config", "jsonschema", "1-0-1") + val configWithDefaultRules = json"""{ "enabled": true, "parameters": { } - }""") - - val externalUri = "http://public-website.com/files/" - val database = "myrules.yml" + }""" + val externalUri = "http://public-website.com/files/" + val database = "myrules.yml" val configWithExternalRules = parse(raw"""{ "enabled": true, "parameters": { "uri": "$externalUri", "database": "$database" } - }""") + }""").toOption.get - "Configuration" | "Custom Rules" | - configWithDefaultRules !! None | + "Configuration" | "Custom Rules" | + configWithDefaultRules !! None | configWithExternalRules !! Some((new URI(externalUri + database), "./ua-parser-rules.yml")) |> { (config, expected) => { @@ -218,8 +214,7 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid currency_convert_config enrichment JSON" should { "successfully construct a CurrencyConversionEnrichment case object" in { - - val currencyConversionEnrichmentJson = parse("""{ + val currencyConversionEnrichmentJson = json"""{ "enabled": true, "parameters": { "accountType": "DEVELOPER", @@ -227,40 +222,41 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "baseCurrency": "EUR", "rateAt": "EOD_PRIOR" } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "currency_conversion_config", "jsonschema", "1-0-0") - - val result = CurrencyConversionEnrichmentConfig.parse(currencyConversionEnrichmentJson, schemaKey) - result must beSuccessful(CurrencyConversionEnrichment(DeveloperAccount, "---", "EUR", "EOD_PRIOR")) - + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "currency_conversion_config", + "jsonschema", + "1-0-0") + val result = + CurrencyConversionEnrichmentConfig.parse(currencyConversionEnrichmentJson, schemaKey) + result must beSuccessful( + CurrencyConversionEnrichment(DeveloperAccount, "---", "EUR", "EOD_PRIOR")) } } "Parsing a valid javascript_script_config enrichment JSON" should { "successfully construct a JavascriptScriptEnrichment case class" in { - val script = s"""|function process(event) { | return []; |} |""".stripMargin - val javascriptScriptEnrichmentJson = { val encoder = new Base64(true) val encoded = new String(encoder.encode(script.getBytes)).trim // Newline being appended by some Base64 versions parse(s"""{ "enabled": true, "parameters": { - "script": "${encoded}" + "script": "$encoded" } - }""") + }""").toOption.get } - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "javascript_script_config", "jsonschema", "1-0-0") - - // val expected = JavascriptScriptEnrichment(JavascriptScriptEnrichmentConfig.compile(script).toOption.get) - + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + "1-0-0") val result = JavascriptScriptEnrichmentConfig.parse(javascriptScriptEnrichmentJson, schemaKey) result must beSuccessful // TODO: check the result's contents by evaluating some JavaScript } @@ -268,19 +264,19 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid event_fingerprint_config enrichment JSON" should { "successfully construct a EventFingerprintEnrichmentConfig case class" in { - - val refererParserJson = parse("""{ + val refererParserJson = json"""{ "enabled": true, "parameters": { "hashAlgorithm": "MD5", "excludeParameters": ["stm"] } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "event_fingerprint_config", "jsonschema", "1-0-0") - + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "event_fingerprint_config", + "jsonschema", + "1-0-0") val expectedExcludedParameters = List("stm") - val result = EventFingerprintEnrichmentConfig.parse(refererParserJson, schemaKey) result must beSuccessful.like { case enr => enr.algorithm("sample") must beEqualTo("5e8ff9bf55ba3508199d22e984129be6") @@ -290,16 +286,17 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing a valid cookie_extractor_config enrichment JSON" should { "successfully construct a CookieExtractorEnrichment case object" in { - - val cookieExtractorEnrichmentJson = parse("""{ + val cookieExtractorEnrichmentJson = json"""{ "enabled": true, "parameters": { "cookies": ["foo", "bar"] } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "cookie_extractor_config", "jsonschema", "1-0-0") - + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "cookie_extractor_config", + "jsonschema", + "1-0-0") val result = CookieExtractorEnrichmentConfig.parse(cookieExtractorEnrichmentJson, schemaKey) result must beSuccessful(CookieExtractorEnrichment(List("foo", "bar"))) } @@ -310,45 +307,53 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D import pii._ val piiPseudonymizerEnrichmentJson = parse("""{ - | "enabled": true, - | "emitEvent": true, - | "parameters": { - | "pii": [ - | { - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "json": { - | "jsonPath": "$.emailAddress", - | "schemaCriterion": "iglu:com.acme/email_sent/jsonschema/1-*-*", - | "field": "contexts" - | } - | } - | ], - | "strategy": { - | "pseudonymize": { - | "hashFunction": "SHA-256", - | "salt": "pepper" - | } - | } - | } - |}""".stripMargin) - + "enabled": true, + "emitEvent": true, + "parameters": { + "pii": [ + { + "pojo": { + "field": "user_id" + } + }, + { + "json": { + "jsonPath": "$.emailAddress", + "schemaCriterion": "iglu:com.acme/email_sent/jsonschema/1-*-*", + "field": "contexts" + } + } + ], + "strategy": { + "pseudonymize": { + "hashFunction": "SHA-256", + "salt": "pepper" + } + } + } + }""").toOption.get val schemaKey = - SchemaKey("com.snowplowanalytics.snowplow.enrichments", "pii_enrichment_config", "jsonschema", "2-0-0") - + SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "pii_enrichment_config", + "jsonschema", + "2-0-0") val result = PiiPseudonymizerEnrichment.parse(piiPseudonymizerEnrichmentJson, schemaKey) result must beSuccessful.like { case piiRes: PiiPseudonymizerEnrichment => { (piiRes.strategy must haveClass[PiiStrategyPseudonymize]) and - (piiRes.strategy.asInstanceOf[PiiStrategyPseudonymize].hashFunction("1234".getBytes("UTF-8")) + (piiRes.strategy + .asInstanceOf[PiiStrategyPseudonymize] + .hashFunction("1234".getBytes("UTF-8")) must_== "03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4") and (piiRes.fieldList.size must_== 2) and (piiRes.fieldList(0) must haveClass[PiiScalar]) and - (piiRes.fieldList(0).asInstanceOf[PiiScalar].fieldMutator must_== ScalarMutators.get("user_id").get) and - (piiRes.fieldList(1).asInstanceOf[PiiJson].fieldMutator must_== JsonMutators.get("contexts").get) and + (piiRes.fieldList(0).asInstanceOf[PiiScalar].fieldMutator must_== ScalarMutators + .get("user_id") + .get) and + (piiRes.fieldList(1).asInstanceOf[PiiJson].fieldMutator must_== JsonMutators + .get("contexts") + .get) and (piiRes .fieldList(1) .asInstanceOf[PiiJson] @@ -362,8 +367,7 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "Parsing an iab_spiders_and_robots_enrichment JSON" should { "successfully construct an IabEnrichment case class" in { - - val iabJson = parse("""{ + val iabJson = json"""{ "enabled": true, "parameters": { "ipFile": { @@ -379,37 +383,36 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "uri": "https://example.com/" } } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow.enrichments", - "iab_spiders_and_robots_enrichment", - "jsonschema", - "1-0-0") - + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "iab_spiders_and_robots_enrichment", + "jsonschema", + "1-0-0") val expected = IabEnrichment( Some( - IabDatabase("ipFile", - new URI("https://example.com/ip_exclude_current_cidr.txt"), - "ip_exclude_current_cidr.txt")), + IabDatabase( + "ipFile", + new URI("https://example.com/ip_exclude_current_cidr.txt"), + "ip_exclude_current_cidr.txt")), Some( - IabDatabase("excludeUseragentFile", - new URI("https://example.com/exclude_current.txt"), - "exclude_current.txt")), + IabDatabase( + "excludeUseragentFile", + new URI("https://example.com/exclude_current.txt"), + "exclude_current.txt")), Some( - IabDatabase("includeUseragentFile", - new URI("https://example.com/include_current.txt"), - "include_current.txt")), + IabDatabase( + "includeUseragentFile", + new URI("https://example.com/include_current.txt"), + "include_current.txt")), true ) - val result = IabEnrichment.parse(iabJson, schemaKey, true) result must beSuccessful(expected) - } "fail if a database file is missing" in { - - val iabJson = parse("""{ + val iabJson = json"""{ "enabled": true, "parameters": { "ipFile": { @@ -425,20 +428,18 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "uri": "https://example.com" } } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow.enrichments", - "iab_spiders_and_robots_enrichment", - "jsonschema", - "1-0-0") - + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "iab_spiders_and_robots_enrichment", + "jsonschema", + "1-0-0") IabEnrichment.parse(iabJson, schemaKey, true) must throwA[NullPointerException] } "fail if the URI to a database file is invalid" in { - - val iabJson = parse("""{ + val iabJson = json"""{ "enabled": true, "parameters": { "ipFile": { @@ -454,16 +455,14 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers with D "uri": "file://foo:{waaat}/" } } - }""") - - val schemaKey = SchemaKey("com.snowplowanalytics.snowplow.enrichments", - "iab_spiders_and_robots_enrichment", - "jsonschema", - "1-0-0") - + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "iab_spiders_and_robots_enrichment", + "jsonschema", + "1-0-0") val result = IabEnrichment.parse(iabJson, schemaKey, true) result must beFailing - } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala index 0499e018c..7f3c4d484 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala @@ -29,7 +29,9 @@ class EventFingerprintEnrichmentSpec extends Specification with ValidationMatche """ val standardConfig = - EventFingerprintEnrichment(EventFingerprintEnrichmentConfig.getAlgorithm("MD5").toOption.get, List("stm", "eid")) + EventFingerprintEnrichment( + EventFingerprintEnrichmentConfig.getAlgorithm("MD5").toOption.get, + List("stm", "eid")) def e1 = { val config = EventFingerprintEnrichment( @@ -39,16 +41,15 @@ class EventFingerprintEnrichmentSpec extends Specification with ValidationMatche config.getEventFingerprint( Map( - "stm" -> "1000000000000", - "e" -> "se", + "stm" -> "1000000000000", + "e" -> "se", "se_ac" -> "buy" )) must_== "15" } def e2 = { - val initialVersion = Map( - "e" -> "se", + "e" -> "se", "se_ac" -> "action", "se_ca" -> "category", "se_pr" -> "property" @@ -58,26 +59,28 @@ class EventFingerprintEnrichmentSpec extends Specification with ValidationMatche "se_ca" -> "category", "se_ac" -> "action", "se_pr" -> "property", - "e" -> "se" + "e" -> "se" ) - standardConfig.getEventFingerprint(permutedVersion) must_== standardConfig.getEventFingerprint(initialVersion) + standardConfig.getEventFingerprint(permutedVersion) must_== standardConfig.getEventFingerprint( + initialVersion) } def e3 = { val initialVersion = Map( - "stm" -> "1000000000000", - "eid" -> "123e4567-e89b-12d3-a456-426655440000", - "e" -> "se", + "stm" -> "1000000000000", + "eid" -> "123e4567-e89b-12d3-a456-426655440000", + "e" -> "se", "se_ac" -> "buy" ) val delayedVersion = Map( - "stm" -> "9999999999999", - "e" -> "se", + "stm" -> "9999999999999", + "e" -> "se", "se_ac" -> "buy" ) - standardConfig.getEventFingerprint(delayedVersion) must_== standardConfig.getEventFingerprint(initialVersion) + standardConfig.getEventFingerprint(delayedVersion) must_== standardConfig.getEventFingerprint( + initialVersion) } def e4 = { @@ -86,7 +89,8 @@ class EventFingerprintEnrichmentSpec extends Specification with ValidationMatche ) val overlappingVersion = Map("prefi" -> "xsuffix") - standardConfig.getEventFingerprint(initialVersion) should not be standardConfig.getEventFingerprint(initialVersion) + standardConfig.getEventFingerprint(initialVersion) should not be standardConfig + .getEventFingerprint(initialVersion) } def e5 = { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala index 5fd78c97c..a193fb7f4 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala @@ -11,8 +11,8 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry -import org.json4s._ -import org.json4s.jackson.JsonMethods._ +import io.circe._ +import io.circe.literal._ import org.specs2.Specification import org.specs2.scalaz._ @@ -26,28 +26,21 @@ class HttpHeaderExtractorEnrichmentSpec extends Specification with ValidationMat def e1 = { val expected = List( - """{"schema":"iglu:org.ietf/http_header/jsonschema/1-0-0","data":{"name":"X-Forwarded-For","value":"129.78.138.66, 129.78.64.103"}}""" + json"""{"schema":"iglu:org.ietf/http_header/jsonschema/1-0-0","data":{"name":"X-Forwarded-For","value":"129.78.138.66, 129.78.64.103"}}""" ) - HttpHeaderExtractorEnrichment("X-Forwarded-For") - .extract(List("X-Forwarded-For: 129.78.138.66, 129.78.64.103")) - .map(h => compact(render(h))) must_== expected.map(e => compact(render(parse(e)))) + .extract(List("X-Forwarded-For: 129.78.138.66, 129.78.64.103")) must_== expected } def e2 = { val expected = List( - """{"schema":"iglu:org.ietf/http_header/jsonschema/1-0-0","data":{"name":"Accept","value":"text/html"}}""" + json"""{"schema":"iglu:org.ietf/http_header/jsonschema/1-0-0","data":{"name":"Accept","value":"text/html"}}""" ) - - HttpHeaderExtractorEnrichment(".*").extract(List("Accept: text/html")).map(h => compact(render(h))) must_== expected - .map(e => compact(render(parse(e)))) + HttpHeaderExtractorEnrichment(".*").extract(List("Accept: text/html")) must_== expected } def e3 = { - val expected = List.empty[String] - - HttpHeaderExtractorEnrichment(".*") - .extract(Nil) - .map(h => compact(render(h))) must_== expected.map(e => compact(render(parse(e)))) + val expected = List.empty[Json] + HttpHeaderExtractorEnrichment(".*").extract(Nil) must_== expected } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala index 35ffc26e5..1ece4cea2 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala @@ -14,14 +14,19 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.net.URI +import io.circe.literal._ import org.joda.time.DateTime -import org.json4s.jackson.JsonMethods.parse import org.specs2.{ScalaCheck, Specification} import org.specs2.matcher.DataTables import org.specs2.scalaz.ValidationMatchers import scalaz._ -class IabEnrichmentSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class IabEnrichmentSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { + def is = s2""" This is a specification to test the IabEnrichment @@ -40,12 +45,19 @@ class IabEnrichmentSpec extends Specification with DataTables with ValidationMat ) def e1 = - "SPEC NAME" || "USER AGENT" | "IP ADDRESS" | "EXPECTED SPIDER OR ROBOT" | "EXPECTED CATEGORY" | "EXPECTED REASON" | "EXPECTED PRIMARY IMPACT" | - "null UA/IP" !! null ! null ! false ! "BROWSER" ! "PASSED_ALL" ! "NONE" | - "valid UA/IP" !! "Xdroid" ! "192.168.0.1" ! false ! "BROWSER" ! "PASSED_ALL" ! "NONE" | - "valid UA, excluded IP" !! "Mozilla/5.0" ! "192.168.151.21" ! true ! "SPIDER_OR_ROBOT" ! "FAILED_IP_EXCLUDE" ! "UNKNOWN" | - "invalid UA, excluded IP" !! "xonitor" ! "192.168.0.1" ! true ! "SPIDER_OR_ROBOT" ! "FAILED_UA_INCLUDE" ! "UNKNOWN" |> { - (_, userAgent, ipAddress, expectedSpiderOrRobot, expectedCategory, expectedReason, expectedPrimaryImpact) => + "SPEC NAME" || "USER AGENT" | "IP ADDRESS" | "EXPECTED SPIDER OR ROBOT" | "EXPECTED CATEGORY" | "EXPECTED REASON" | "EXPECTED PRIMARY IMPACT" | + "null UA/IP" !! null ! null ! false ! "BROWSER" ! "PASSED_ALL" ! "NONE" | + "valid UA/IP" !! "Xdroid" ! "192.168.0.1" ! false ! "BROWSER" ! "PASSED_ALL" ! "NONE" | + "valid UA, excluded IP" !! "Mozilla/5.0" ! "192.168.151.21" ! true ! "SPIDER_OR_ROBOT" ! "FAILED_IP_EXCLUDE" ! "UNKNOWN" | + "invalid UA, excluded IP" !! "xonitor" ! "192.168.0.1" ! true ! "SPIDER_OR_ROBOT" ! "FAILED_UA_INCLUDE" ! "UNKNOWN" |> { + ( + _, + userAgent, + ipAddress, + expectedSpiderOrRobot, + expectedCategory, + expectedReason, + expectedPrimaryImpact) => { validConfig.performCheck(userAgent, ipAddress, DateTime.now()) must beLike { case Success(check) => @@ -64,18 +76,19 @@ class IabEnrichmentSpec extends Specification with DataTables with ValidationMat validConfig.getIabContext(None, None, None) must beFailing def e4 = { - val responseJson = parse(""" - |{ - | "schema": "iglu:com.iab.snowplow/spiders_and_robots/jsonschema/1-0-0", - | "data": { - | "spiderOrRobot": false, - | "category": "BROWSER", - | "reason": "PASSED_ALL", - | "primaryImpact": "NONE" - | } - |} - """.stripMargin) - validConfig.getIabContext(Some("Xdroid"), Some("192.168.0.1"), Some(DateTime.now())) must beSuccessful(responseJson) + val responseJson = json""" + { + "schema": "iglu:com.iab.snowplow/spiders_and_robots/jsonschema/1-0-0", + "data": { + "spiderOrRobot": false, + "category": "BROWSER", + "reason": "PASSED_ALL", + "primaryImpact": "NONE" + } + } + """ + validConfig.getIabContext(Some("Xdroid"), Some("192.168.0.1"), Some(DateTime.now())) must beSuccessful( + responseJson) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala index 91598c513..a67bb2f31 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala @@ -21,7 +21,11 @@ import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -class IpLookupsEnrichmentSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class IpLookupsEnrichmentSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the IpLookupsEnrichment extractIpInformation should correctly extract location data from IP addresses where possible $e1 @@ -40,12 +44,12 @@ class IpLookupsEnrichmentSpec extends Specification with DataTables with Validat ) def e1 = - "SPEC NAME" || "IP ADDRESS" | "EXPECTED LOCATION" | - "blank IP address" !! "" ! Some(Failure("AddressNotFoundException")) | - "null IP address" !! null ! Some(Failure("AddressNotFoundException")) | + "SPEC NAME" || "IP ADDRESS" | "EXPECTED LOCATION" | + "blank IP address" !! "" ! Some(Failure("AddressNotFoundException")) | + "null IP address" !! null ! Some(Failure("AddressNotFoundException")) | "invalid IP address #1" !! "localhost" ! Some(Failure("AddressNotFoundException")) | "invalid IP address #2" !! "hello" ! Some(Failure("UnknownHostException")) | - "valid IP address" !! "175.16.199.0" ! + "valid IP address" !! "175.16.199.0" ! IpLocation( // Taken from scala-maxmind-geoip. See that test suite for other valid IP addresses countryCode = "CN", countryName = "China", @@ -71,15 +75,20 @@ class IpLookupsEnrichmentSpec extends Specification with DataTables with Validat metroCode = None, regionName = Some("Jilin Sheng") ).success.some |> { (_, ipAddress, expected) => - config.extractIpInformation(ipAddress).ipLocation.map(_.leftMap(_.getClass.getSimpleName)) must_== expected + config + .extractIpInformation(ipAddress) + .ipLocation + .map(_.leftMap(_.getClass.getSimpleName)) must_== expected } - def e2 = config.extractIpInformation("70.46.123.145").isp must_== "FDN Communications".success.some + def e2 = + config.extractIpInformation("70.46.123.145").isp must_== "FDN Communications".success.some def e3 = config.filesToCache must_== Nil val configRemote = IpLookupsEnrichment( - Some(("geo", new URI("http://public-website.com/files/GeoLite2-City.mmdb"), "GeoLite2-City.mmdb")), + Some( + ("geo", new URI("http://public-website.com/files/GeoLite2-City.mmdb"), "GeoLite2-City.mmdb")), Some(("isp", new URI("s3://private-bucket/files/GeoIP2-ISP.mmdb"), "GeoIP2-ISP.mmdb")), None, None, diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala index ea5951e7c..5506fab89 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala @@ -13,16 +13,13 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry +import io.circe.literal._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers -import org.json4s._ -import org.json4s.jackson.JsonMethods._ import outputs.EnrichedEvent -/** - * Tests the anonymzeIp function - */ +/** Tests the anonymzeIp function */ class JavascriptScriptEnrichmentSpec extends Specification with ValidationMatchers { def is = s2""" This is a specification to test the JavascriptScriptEnrichment @@ -56,7 +53,7 @@ class JavascriptScriptEnrichmentSpec extends Specification with ValidationMatche def buildEvent(appId: String): EnrichedEvent = { val e = new EnrichedEvent() e.platform = "server" - e.app_id = appId + e.app_id = appId e } @@ -67,18 +64,16 @@ class JavascriptScriptEnrichmentSpec extends Specification with ValidationMatche def e2 = { val event = buildEvent("guess") - val actual = PreparedEnrichment.process(event) actual must beFailing } def e3 = { val event = buildEvent("secret") - - val actual = PreparedEnrichment.process(event) - val expected = """{"schema":"iglu:com.acme/foo/jsonschema/1-0-0","data":{"appIdUpper":"SECRET"}}""" - - actual must beSuccessful.like { case head :: Nil => compact(render(head)) must_== compact(render(parse(expected))) } + val actual = PreparedEnrichment.process(event) + val expected = + json"""{"schema":"iglu:com.acme/foo/jsonschema/1-0-0","data":{"appIdUpper":"SECRET"}}""" + actual must beSuccessful.like { case head :: Nil => head must_== expected } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala index cff86371c..1656bc7f6 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala @@ -20,10 +20,8 @@ import org.specs2.matcher.DataTables /** * A small selection of tests partially borrowed from referer-parser. - * * This is a very imcomplete set - more a tripwire than an exhaustive test. * Please see referer-parser's test suite for the full set of tests: - * * https://github.com/snowplow/referer-parser/tree/master/java-scala/src/test/scala/com/snowplowanalytics/refererparser/scala */ class ExtractRefererDetailsSpec extends Specification with DataTables { @@ -36,24 +34,26 @@ class ExtractRefererDetailsSpec extends Specification with DataTables { val PageHost = "www.snowplowanalytics.com" def e1 = - "SPEC NAME" || "REFERER URI" | "REFERER MEDIUM" | "REFERER SOURCE" | "REFERER TERM" | - "Google search" !! "http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari" ! Medium.Search ! Some( - "Google") ! Some("gateway oracle cards denise linn") | - "Facebook social" !! "http://www.facebook.com/l.php?u=http%3A%2F%2Fwww.psychicbazaar.com&h=yAQHZtXxS&s=1" ! Medium.Social ! Some( - "Facebook") ! None | - "Yahoo! Mail" !! "http://36ohk6dgmcd1n-c.c.yom.mail.yahoo.net/om/api/1.0/openmail.app.invoke/36ohk6dgmcd1n/11/1.0.35/us/en-US/view.html/0" ! Medium.Email ! Some( - "Yahoo! Mail") ! None | + "SPEC NAME" || "REFERER URI" | "REFERER MEDIUM" | "REFERER SOURCE" | "REFERER TERM" | + "Google search" !! "http://www.google.com/search?q=gateway+oracle+cards+denise+linn&hl=en&client=safari" ! Medium.Search ! Some( + "Google") ! Some("gateway oracle cards denise linn") | + "Facebook social" !! "http://www.facebook.com/l.php?u=http%3A%2F%2Fwww.psychicbazaar.com&h=yAQHZtXxS&s=1" ! Medium.Social ! Some( + "Facebook") ! None | + "Yahoo! Mail" !! "http://36ohk6dgmcd1n-c.c.yom.mail.yahoo.net/om/api/1.0/openmail.app.invoke/36ohk6dgmcd1n/11/1.0.35/us/en-US/view.html/0" ! Medium.Email ! Some( + "Yahoo! Mail") ! None | "Internal referer" !! "https://www.snowplowanalytics.com/account/profile" ! Medium.Internal ! None ! None | - "Custom referer" !! "https://www.internaldomain.com/path" ! Medium.Internal ! None ! None | - "Unknown referer" !! "http://www.spyfu.com/domain.aspx?d=3897225171967988459" ! Medium.Unknown ! None ! None |> { + "Custom referer" !! "https://www.internaldomain.com/path" ! Medium.Internal ! None ! None | + "Unknown referer" !! "http://www.spyfu.com/domain.aspx?d=3897225171967988459" ! Medium.Unknown ! None ! None |> { (_, refererUri, medium, source, term) => RefererParserEnrichment(List("www.internaldomain.com")) - .extractRefererDetails(new URI(refererUri), PageHost) must_== Some(Referer(medium, source, term)) + .extractRefererDetails(new URI(refererUri), PageHost) must_== Some( + Referer(medium, source, term)) } def e2 = RefererParserEnrichment(List()).extractRefererDetails( - new URI("http://www.google.com/search?q=%0Agateway%09oracle%09cards%09denise%09linn&hl=en&client=safari"), + new URI( + "http://www.google.com/search?q=%0Agateway%09oracle%09cards%09denise%09linn&hl=en&client=safari"), PageHost) must_== Some( Referer(Medium.Search, Some("Google"), Some("gateway oracle cards denise linn"))) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala index e3a6d7e09..a7cab7876 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala @@ -13,50 +13,49 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.net.URI -import org.json4s._ -import org.json4s.jackson.JsonMethods._ +import io.circe.literal._ import org.specs2.matcher.DataTables import org.specs2.scalaz._ -class UaParserEnrichmentSpec extends org.specs2.mutable.Specification with ValidationMatchers with DataTables { +class UaParserEnrichmentSpec + extends org.specs2.mutable.Specification + with ValidationMatchers + with DataTables { val mobileSafariUserAgent = "Mozilla/5.0 (iPhone; CPU iPhone OS 5_1_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B206 Safari/7534.48.3" val mobileSafariJson = - """{"schema":"iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0","data":{"useragentFamily":"Mobile Safari","useragentMajor":"5","useragentMinor":"1","useragentPatch":null,"useragentVersion":"Mobile Safari 5.1","osFamily":"iOS","osMajor":"5","osMinor":"1","osPatch":"1","osPatchMinor":null,"osVersion":"iOS 5.1.1","deviceFamily":"iPhone"}}""" + json"""{"schema":"iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0","data":{"useragentFamily":"Mobile Safari","useragentMajor":"5","useragentMinor":"1","useragentPatch":null,"useragentVersion":"Mobile Safari 5.1","osFamily":"iOS","osMajor":"5","osMinor":"1","osPatch":"1","osPatchMinor":null,"osVersion":"iOS 5.1.1","deviceFamily":"iPhone"}}""" val safariUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25" val safariJson = - """{"schema":"iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0","data":{"useragentFamily":"Safari","useragentMajor":"8","useragentMinor":"0","useragentPatch":null,"useragentVersion":"Safari 8.0","osFamily":"Mac OS X","osMajor":"10","osMinor":"10","osPatch":null,"osPatchMinor":null,"osVersion":"Mac OS X 10.10","deviceFamily":"Other"}}""" + json"""{"schema":"iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0","data":{"useragentFamily":"Safari","useragentMajor":"8","useragentMinor":"0","useragentPatch":null,"useragentVersion":"Safari 8.0","osFamily":"Mac OS X","osMajor":"10","osMinor":"10","osPatch":null,"osPatchMinor":null,"osVersion":"Mac OS X 10.10","deviceFamily":"Other"}}""" // The URI is irrelevant here, but the local file name needs to point to our test resource val testRulefile = getClass.getResource("uap-test-rules.yml").toURI.getPath - val customRules = (new URI("s3://private-bucket/files/uap-rules.yml"), testRulefile) + val customRules = (new URI("s3://private-bucket/files/uap-rules.yml"), testRulefile) val testAgentJson = - """{"schema":"iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0","data":{"useragentFamily":"UAP Test Family","useragentMajor":null,"useragentMinor":null,"useragentPatch":null,"useragentVersion":"UAP Test Family","osFamily":"UAP Test OS","osMajor":null,"osMinor":null,"osPatch":null,"osPatchMinor":null,"osVersion":"UAP Test OS","deviceFamily":"UAP Test Device"}}""" + json"""{"schema":"iglu:com.snowplowanalytics.snowplow/ua_parser_context/jsonschema/1-0-0","data":{"useragentFamily":"UAP Test Family","useragentMajor":null,"useragentMinor":null,"useragentPatch":null,"useragentVersion":"UAP Test Family","osFamily":"UAP Test OS","osMajor":null,"osMinor":null,"osPatch":null,"osPatchMinor":null,"osVersion":"UAP Test OS","deviceFamily":"UAP Test Device"}}""" "useragent parser enrichment" should { "report files needing to be cached" in { - "Custom Rules" | "Cached Files" | - None !! List.empty | + "Custom Rules" | "Cached Files" | + None !! List.empty | Some(customRules) !! List(customRules) |> { (rules, cachedFiles) => - { - UaParserEnrichment(rules).filesToCache must_== cachedFiles - } + UaParserEnrichment(rules).filesToCache must_== cachedFiles } } } "useragent parser" should { "parse useragent according to configured rules" in { - "Custom Rules" | "Input UserAgent" | "Parsed UserAgent" | - None !! mobileSafariUserAgent !! mobileSafariJson | - None !! safariUserAgent !! safariJson | + "Custom Rules" | "Input UserAgent" | "Parsed UserAgent" | + None !! mobileSafariUserAgent !! mobileSafariJson | + None !! safariUserAgent !! safariJson | Some(customRules) !! mobileSafariUserAgent !! testAgentJson |> { (rules, input, expected) => - { - UaParserEnrichment(rules).extractUserAgent(input) must beSuccessful.like { - case a => compact(render(a)) must_== compact(render(parse(expected))) - } + UaParserEnrichment(rules).extractUserAgent(input) must beSuccessful.like { + case a => + a must_== expected } } } @@ -66,13 +65,11 @@ class UaParserEnrichmentSpec extends org.specs2.mutable.Specification with Valid "useragent parser" should { "report initialization error" in { - "Custom Rules" | "Input UserAgent" | "Parsed UserAgent" | + "Custom Rules" | "Input UserAgent" | "Parsed UserAgent" | Some(badRulefile) !! mobileSafariUserAgent !! "Failed to initialize ua parser" |> { (rules, input, errorPrefix) => - { - UaParserEnrichment(rules).extractUserAgent(input) must beFailing.like { - case a => a must startWith(errorPrefix) - } + UaParserEnrichment(rules).extractUserAgent(input) must beFailing.like { + case a => a must startWith(errorPrefix) } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala index b7c76a0a7..dde8108d3 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala @@ -14,35 +14,42 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import org.specs2.matcher.DataTables import org.specs2.scalaz._ -class UserAgentUtilsEnrichmentSpec extends org.specs2.mutable.Specification with ValidationMatchers with DataTables { +class UserAgentUtilsEnrichmentSpec + extends org.specs2.mutable.Specification + with ValidationMatchers + with DataTables { "useragent parser" should { "parse useragent" in { "SPEC NAME" || "Input UserAgent" | "Browser name" | "Browser family" | "Browser version" | "Browser type" | "Browser rendering enging" | "OS fields" | "Device type" | "Device is mobile" |> - "Safari spec" !! "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36" ! "Chrome 33" ! "Chrome" ! Some("33.0.1750.152") ! "Browser" ! "WEBKIT" ! (("Mac OS X", "Mac OS X", "Apple Inc.")) ! "Computer" ! false | - "IE spec" !! "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0" ! "Internet Explorer 11" ! "Internet Explorer" ! Some("11.0") ! "Browser" ! "TRIDENT" ! (("Windows 7", "Windows", "Microsoft Corporation")) ! "Computer" ! false | { + "Safari spec" !! "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36" ! "Chrome 33" ! "Chrome" ! Some( + "33.0.1750.152") ! "Browser" ! "WEBKIT" ! (("Mac OS X", "Mac OS X", "Apple Inc.")) ! "Computer" ! false | + "IE spec" !! "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0" ! "Internet Explorer 11" ! "Internet Explorer" ! Some( + "11.0") ! "Browser" ! "TRIDENT" ! (("Windows 7", "Windows", "Microsoft Corporation")) ! "Computer" ! false | { - (_, - input, - browserName, - browserFamily, - browserVersion, - browserType, - browserRenderEngine, - osFields, - deviceType, - deviceIsMobile) => + ( + _, + input, + browserName, + browserFamily, + browserVersion, + browserType, + browserRenderEngine, + osFields, + deviceType, + deviceIsMobile) => { - val expected = ClientAttributes(browserName, - browserFamily, - browserVersion, - browserType, - browserRenderEngine, - osFields._1, - osFields._2, - osFields._3, - deviceType, - deviceIsMobile) + val expected = ClientAttributes( + browserName, + browserFamily, + browserVersion, + browserType, + browserRenderEngine, + osFields._1, + osFields._2, + osFields._3, + deviceType, + deviceIsMobile) UserAgentUtilsEnrichment.extractClientAttributes(input) must beSuccessful(expected) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala index e4fb66f7f..fe4cf2c87 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala @@ -1,4 +1,5 @@ -/**Copyright (c) 2012-2019 Snowplow Analytics Ltd. All rights reserved. +/** + * Copyright (c) 2012-2019 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -14,9 +15,9 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry import java.lang.{Float => JFloat} import com.snowplowanalytics.iglu.client.SchemaKey +import io.circe.generic.auto._ +import io.circe.literal._ import org.joda.time.DateTime -import org.json4s._ -import org.json4s.jackson.JsonMethods.parse import org.specs2.Specification object WeatherEnrichmentSpec { @@ -40,12 +41,12 @@ class WeatherEnrichmentSpec extends Specification { lazy val validAppKey = sys.env .get(OwmApiKey) - .getOrElse( - throw new IllegalStateException(s"No ${OwmApiKey} environment variable found, test should have been skipped")) + .getOrElse(throw new IllegalStateException( + s"No $OwmApiKey environment variable found, test should have been skipped")) object invalidEvent { - var lat: JFloat = 70.98224f - var lon: JFloat = 70.98224f + var lat: JFloat = 70.98224f + var lon: JFloat = 70.98224f var time: DateTime = null } @@ -56,28 +57,36 @@ class WeatherEnrichmentSpec extends Specification { } def e1 = { - val enr = WeatherEnrichment("KEY", 5200, 1, "history.openweathermap.org", 10) - val stamp = enr.getWeatherContext(Option(invalidEvent.lat), Option(invalidEvent.lon), Option(invalidEvent.time)) + val enr = WeatherEnrichment("KEY", 5200, 1, "history.openweathermap.org", 10) + val stamp = enr.getWeatherContext( + Option(invalidEvent.lat), + Option(invalidEvent.lon), + Option(invalidEvent.time)) stamp.toEither must beLeft.like { case e => e must contain("tstamp: None") } } - def e2 = WeatherEnrichment("KEY", 0, 1, "history.openweathermap.org", 5) must not(throwA[IllegalArgumentException]) + def e2 = + WeatherEnrichment("KEY", 0, 1, "history.openweathermap.org", 5) must not( + throwA[IllegalArgumentException]) def e3 = { - val enr = WeatherEnrichment(validAppKey, 5200, 1, "history.openweathermap.org", 10) - val stamp = enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) + val enr = WeatherEnrichment(validAppKey, 5200, 1, "history.openweathermap.org", 10) + val stamp = + enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) stamp.toEither must beRight } def e4 = { - val enr = WeatherEnrichment("KEY", 5200, 1, "history.openweathermap.org", 10) - val stamp = enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) + val enr = WeatherEnrichment("KEY", 5200, 1, "history.openweathermap.org", 10) + val stamp = + enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) stamp.toEither must beLeft.like { case e => e must contain("AuthorizationError") } } def e5 = { - val enr = WeatherEnrichment(validAppKey, 5200, 1, "history.openweathermap.org", 15) - val stamp = enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) + val enr = WeatherEnrichment(validAppKey, 5200, 1, "history.openweathermap.org", 15) + val stamp = + enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) stamp.toEither must beRight.like { case weather: JValue => { val temp = weather.findField { case JField("humidity", _) => true; case _ => false } @@ -87,42 +96,46 @@ class WeatherEnrichmentSpec extends Specification { } def e6 = { - val configJson = parse(""" - |{ - | "enabled": true, - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "weather_enrichment_config", - | "parameters": { - | "apiKey": "{{KEY}}", - | "cacheSize": 5100, - | "geoPrecision": 1, - | "apiHost": "history.openweathermap.org", - | "timeout": 5 - | } - |} - """.stripMargin) + val configJson = json""" + { + "enabled": true, + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "weather_enrichment_config", + "parameters": { + "apiKey": "{{KEY}}", + "cacheSize": 5100, + "geoPrecision": 1, + "apiHost": "history.openweathermap.org", + "timeout": 5 + } + } + """ val config = WeatherEnrichmentConfig.parse( configJson, - SchemaKey("com.snowplowanalytics.snowplow.enrichments", "weather_enrichment_config", "jsonschema", "1-0-0")) + SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "weather_enrichment_config", + "jsonschema", + "1-0-0")) config.toEither must beRight( - WeatherEnrichment(apiKey = "{{KEY}}", - geoPrecision = 1, - cacheSize = 5100, - apiHost = "history.openweathermap.org", - timeout = 5)) + WeatherEnrichment( + apiKey = "{{KEY}}", + geoPrecision = 1, + cacheSize = 5100, + apiHost = "history.openweathermap.org", + timeout = 5)) } def e7 = { - implicit val formats = DefaultFormats - val enr = WeatherEnrichment(validAppKey, 2, 1, "history.openweathermap.org", 15) - val stamp = enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) + val enr = WeatherEnrichment(validAppKey, 2, 1, "history.openweathermap.org", 15) + val stamp = + enr.getWeatherContext(Option(validEvent.lat), Option(validEvent.lon), Option(validEvent.time)) stamp.toEither must beRight.like { // successful request - case weather: JValue => { + case weather: JValue => val e = (weather \ "data").extractOpt[TransformedWeather] e.map(_.dt) must beSome.like { // succesfull transformation case dt => dt must equalTo("2019-05-01T00:00:00.000Z") // closest stamp storing on server } - } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala index 8bb78f1c5..c9fac4e3d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala @@ -13,11 +13,13 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry.apirequest +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{JsonSchemaPair, SchemaKey} -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.{parseJson, prettyJson} -import org.json4s.jackson.JsonMethods.asJsonNode +import io.circe._ +import io.circe.jackson.{circeToJackson, jacksonToCirce} +import io.circe.literal._ +import io.circe.parser._ +import io.circe.syntax._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers import org.specs2.matcher.Matcher @@ -27,7 +29,7 @@ import outputs.EnrichedEvent object ApiRequestEnrichmentIntegrationTest { def continuousIntegration: Boolean = sys.env.get("CI") match { case Some("true") => true - case _ => false + case _ => false } /** @@ -37,11 +39,17 @@ object ApiRequestEnrichmentIntegrationTest { * ApiRequestEnrichment.lookup method */ def createPair(key: SchemaKey, validJson: String): JsonSchemaPair = { - val hierarchy = parseJson( - s"""{"rootId":null,"rootTstamp":null,"refRoot":"events","refTree":["events","${key.name}"],"refParent":"events"}""") - (key, asJsonNode(("data", parseJson(validJson)) ~ - (("hierarchy", hierarchy)) ~ - (("schema", key.toJValue)))) + val hierarchy = parse( + s"""{"rootId":null,"rootTstamp":null,"refRoot":"events","refTree":["events","${key.name}"],"refParent":"events"}""").toOption + .get + ( + key, + circeToJackson( + Json.obj( + "data" := parse(validJson).toOption.get, + "hierarchy" := hierarchy, + "schema" := jacksonToCirce(key.toJsonNode) + ))) } } @@ -56,180 +64,162 @@ class ApiRequestEnrichmentIntegrationTest extends Specification with ValidationM """ object IntegrationTests { - val configuration = parseJson(""" - |{ - | - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "api_request_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "key": "user", - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "key": "client", - | "pojo": { - | "field": "app_id" - | } - | } - | ], - | "api": { - | "http": { - | "method": "GET", - | "uri": "http://localhost:8000/guest/api/{{client}}/{{user}}?format=json", - | "timeout": 5000, - | "authentication": {} - | } - | }, - | "outputs": [{ - | "schema": "iglu:com.acme/unauth/jsonschema/1-0-0", - | "json": { - | "jsonPath": "$" - | } - | }], - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | } - | - """.stripMargin) + val configuration = parse( + """{ + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "api_request_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "key": "user", + "pojo": { + "field": "user_id" + } + }, + { + "key": "client", + "pojo": { + "field": "app_id" + } + } + ], + "api": { + "http": { + "method": "GET", + "uri": "http://localhost:8000/guest/api/{{client}}/{{user}}?format=json", + "timeout": 5000, + "authentication": {} + } + }, + "outputs": [{ + "schema": "iglu:com.acme/unauth/jsonschema/1-0-0", + "json": { + "jsonPath": "$" + } + }], + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get - val correctResultContext = parseJson( - """ - |{ - | "schema": "iglu:com.acme/unauth/jsonschema/1-0-0", - | "data": {"path": "/guest/api/lookup-test/snowplower?format=json", "message": "unauthorized", "method": "GET"} - |} - """.stripMargin) + val correctResultContext = json"""{ + "schema": "iglu:com.acme/unauth/jsonschema/1-0-0", + "data": {"path": "/guest/api/lookup-test/snowplower?format=json", "message": "unauthorized", "method": "GET"} + }""" - val configuration2 = parseJson( - """ - |{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "api_request_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "key": "user", - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "key": "client", - | "pojo": { - | "field": "app_id" - | } - | }, - | { - | "key": "jobflow", - | "json": { - | "field": "unstruct_event", - | "jsonPath": "$.jobflow_id", - | "schemaCriterion": "iglu:com.snowplowanalytics.monitoring.batch/emr_job_status/jsonschema/*-*-*" - | } - | }, - | { - | "key": "latitude", - | "json": { - | "field": "contexts", - | "jsonPath": "$.latitude", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0" - | } - | }, - | { - | "key": "datetime", - | "json": { - | "field": "derived_contexts", - | "jsonPath": "$.dt", - | "schemaCriterion": "iglu:org.openweathermap/weather/jsonschema/1-*-*" - | } - | }], - | "api": { - | "http": { - | "method": "POST", - | "uri": "http://localhost:8000/api/{{client}}/{{user}}/{{ jobflow }}/{{latitude}}?date={{ datetime }}", - | "timeout": 5000, - | "authentication": { - | "httpBasic": { - | "username": "snowplower", - | "password": "supersecret" - | } - | } - | } - | }, - | "outputs": [{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "json": { - | "jsonPath": "$.data.lookupArray[0]" - | } - | }, { - | "schema": "iglu:com.acme/onlypath/jsonschema/1-0-0", - | "json": { - | "jsonPath": "$.data.lookupArray[1]" - | } - | }], - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | } - | - """.stripMargin) + val configuration2 = parse( + """{ + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "api_request_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "key": "user", + "pojo": { + "field": "user_id" + } + }, + { + "key": "client", + "pojo": { + "field": "app_id" + } + }, + { + "key": "jobflow", + "json": { + "field": "unstruct_event", + "jsonPath": "$.jobflow_id", + "schemaCriterion": "iglu:com.snowplowanalytics.monitoring.batch/emr_job_status/jsonschema/*-*-*" + } + }, + { + "key": "latitude", + "json": { + "field": "contexts", + "jsonPath": "$.latitude", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0" + } + }, + { + "key": "datetime", + "json": { + "field": "derived_contexts", + "jsonPath": "$.dt", + "schemaCriterion": "iglu:org.openweathermap/weather/jsonschema/1-*-*" + } + }], + "api": { + "http": { + "method": "POST", + "uri": "http://localhost:8000/api/{{client}}/{{user}}/{{ jobflow }}/{{latitude}}?date={{ datetime }}", + "timeout": 5000, + "authentication": { + "httpBasic": { + "username": "snowplower", + "password": "supersecret" + } + } + } + }, + "outputs": [{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "json": { + "jsonPath": "$.data.lookupArray[0]" + } + }, { + "schema": "iglu:com.acme/onlypath/jsonschema/1-0-0", + "json": { + "jsonPath": "$.data.lookupArray[1]" + } + }], + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get // NOTE: akka-http 1.0 was sending "2014-11-10T08:38:30.000Z" as is with ':', this behavior was changed in 2.0 - val correctResultContext2 = parseJson( - """ - |{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "data": {"path": "/api/lookup+test/snowplower/j-ZKIY4CKQRX72/32.1?date=2014-11-10T08%3A38%3A30.000Z","method": "POST", "auth_header": "snowplower:supersecret", "request": 1} - |} - """.stripMargin) + val correctResultContext2 = json"""{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "data": {"path": "/api/lookup+test/snowplower/j-ZKIY4CKQRX72/32.1?date=2014-11-10T08%3A38%3A30.000Z","method": "POST", "auth_header": "snowplower:supersecret", "request": 1} + }""" - val correctResultContext3 = parseJson( - """ - |{ - | "schema": "iglu:com.acme/onlypath/jsonschema/1-0-0", - | "data": {"path": "/api/lookup+test/snowplower/j-ZKIY4CKQRX72/32.1?date=2014-11-10T08%3A38%3A30.000Z", "request": 1} - |} - """.stripMargin) + val correctResultContext3 = json"""{ + "schema": "iglu:com.acme/onlypath/jsonschema/1-0-0", + "data": {"path": "/api/lookup+test/snowplower/j-ZKIY4CKQRX72/32.1?date=2014-11-10T08%3A38%3A30.000Z", "request": 1} + }""" // Usual self-describing instance - val weatherContext = parseJson( - """ - |{ - | "schema": "iglu:org.openweathermap/weather/jsonschema/1-0-0", - | "data": { - | "clouds": { - | "all": 0 - | }, - | "dt": "2014-11-10T08:38:30.000Z", - | "main": { - | "grnd_level": 1021.91, - | "humidity": 90, - | "pressure": 1021.91, - | "sea_level": 1024.77, - | "temp": 301.308, - | "temp_max": 301.308, - | "temp_min": 301.308 - | }, - | "weather": [ { "description": "Sky is Clear", "icon": "01d", "id": 800, "main": "Clear" } ], - | "wind": { - | "deg": 190.002, - | "speed": 4.39 - | } - |} - |} - """.stripMargin).asInstanceOf[JObject] + val weatherContext = json"""{ + "schema": "iglu:org.openweathermap/weather/jsonschema/1-0-0", + "data": { + "clouds": { + "all": 0 + }, + "dt": "2014-11-10T08:38:30.000Z", + "main": { + "grnd_level": 1021.91, + "humidity": 90, + "pressure": 1021.91, + "sea_level": 1024.77, + "temp": 301.308, + "temp_max": 301.308, + "temp_min": 301.308 + }, + "weather": [ { "description": "Sky is Clear", "icon": "01d", "id": 800, "main": "Clear" } ], + "wind": { + "deg": 190.002, + "speed": 4.39 + } + } + }""" // JsonSchemaPair built by Shredder val customContexts = createPair( @@ -249,54 +239,63 @@ class ApiRequestEnrichmentIntegrationTest extends Specification with ValidationM } val SCHEMA_KEY = - SchemaKey("com.snowplowanalytics.snowplow.enrichments", "api_request_enrichment_config", "jsonschema", "1-0-0") + SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "api_request_enrichment_config", + "jsonschema", + "1-0-0") /** * Helper matcher to print JSON */ - def beJson(expected: JValue): Matcher[JValue] = { actual: JValue => - (actual == expected, "actual:\n" + prettyJson(actual) + "\n expected:\n" + prettyJson(expected) + "\n") + def beJson(expected: Json): Matcher[Json] = { actual: Json => + (actual == expected, "actual:\n" + actual.spaces2 + "\n expected:\n" + expected.spaces2 + "\n") } def e1 = { val config = ApiRequestEnrichmentConfig.parse(IntegrationTests.configuration, SCHEMA_KEY) - val event = new EnrichedEvent + val event = new EnrichedEvent event.setApp_id("lookup-test") event.setUser_id("snowplower") val context = config.flatMap(_.lookup(event, Nil, Nil, Nil)) context must beSuccessful.like { - case context => context must contain(IntegrationTests.correctResultContext) and (context must have size (1)) + case context => + context must contain(IntegrationTests.correctResultContext) and (context must have size (1)) } } def e2 = { val config = ApiRequestEnrichmentConfig.parse(IntegrationTests.configuration2, SCHEMA_KEY) - val event = new EnrichedEvent + val event = new EnrichedEvent event.setApp_id("lookup test") event.setUser_id("snowplower") // Fill cache config.flatMap( - _.lookup(event, - List(IntegrationTests.weatherContext), - List(IntegrationTests.customContexts), - List(IntegrationTests.unstructEvent))) + _.lookup( + event, + List(IntegrationTests.weatherContext), + List(IntegrationTests.customContexts), + List(IntegrationTests.unstructEvent))) config.flatMap( - _.lookup(event, - List(IntegrationTests.weatherContext), - List(IntegrationTests.customContexts), - List(IntegrationTests.unstructEvent))) + _.lookup( + event, + List(IntegrationTests.weatherContext), + List(IntegrationTests.customContexts), + List(IntegrationTests.unstructEvent))) val context = config.flatMap( - _.lookup(event, - List(IntegrationTests.weatherContext), - List(IntegrationTests.customContexts), - List(IntegrationTests.unstructEvent))) + _.lookup( + event, + List(IntegrationTests.weatherContext), + List(IntegrationTests.customContexts), + List(IntegrationTests.unstructEvent))) context must beSuccessful.like { case context => - context must contain(beJson(IntegrationTests.correctResultContext2), - beJson(IntegrationTests.correctResultContext3)) and (context must have size (2)) + context must contain( + beJson(IntegrationTests.correctResultContext2), + beJson(IntegrationTests.correctResultContext3)) and (context must have size (2)) } and { config must beSuccessful.like { case c => c.cache.actualLoad must beEqualTo(1) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala index a1ba7abd5..579f54567 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala @@ -13,16 +13,17 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry.apirequest +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.JsonSchemaPair import com.snowplowanalytics.iglu.client.SchemaKey +import io.circe.jackson.circeToJackson +import io.circe.literal._ +import io.circe.parser._ import org.specs2.Specification import org.specs2.mock.Mockito import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ -import org.json4s._ -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.parseJson import outputs.EnrichedEvent @@ -36,7 +37,11 @@ class ApiRequestEnrichmentSpec extends Specification with ValidationMatchers wit """" val SCHEMA_KEY = - SchemaKey("com.snowplowanalytics.snowplow.enrichments", "api_request_enrichment_config", "jsonschema", "1-0-0") + SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "api_request_enrichment_config", + "jsonschema", + "1-0-0") def e1 = { val inputs = List( @@ -45,222 +50,226 @@ class ApiRequestEnrichmentSpec extends Specification with ValidationMatchers wit "userSession", pojo = None, json = Some( - JsonInput("contexts", "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", "$.userId"))), + JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "$.userId"))), Input("client", pojo = Some(PojoInput("app_id")), json = None) ) val api = - HttpApi("GET", - "http://api.acme.com/users/{{client}}/{{user}}?format=json", - 1000, - Authentication(Some(HttpBasic(Some("xxx"), None)))) + HttpApi( + "GET", + "http://api.acme.com/users/{{client}}/{{user}}?format=json", + 1000, + Authentication(Some(HttpBasic(Some("xxx"), None)))) val apiSpy = spy(api) val output = Output("iglu:com.acme/user/jsonschema/1-0-0", Some(JsonOutput("$.record"))) - val cache = Cache(3000, 60) + val cache = Cache(3000, 60) val config = ApiRequestEnrichment(inputs, apiSpy, List(output), cache) val fakeEnrichedEvent = new EnrichedEvent { - app_id = "some-fancy-app-id" + app_id = "some-fancy-app-id" user_id = "some-fancy-user-id" } val clientSession: JsonSchemaPair = ( - SchemaKey(vendor = "com.snowplowanalytics.snowplow", - name = "client_session", - format = "jsonschema", - version = "1-0-1"), - JsonMethods.asJsonNode(parseJson("""|{ - | "data": { - | "userId": "some-fancy-user-session-id", - | "sessionId": "42c8a55b-c0c2-4749-b9ac-09bb0d17d000", - | "sessionIndex": 1, - | "previousSessionId": null, - | "storageMechanism": "COOKIE_1" - | } - |}""".stripMargin)) + SchemaKey( + vendor = "com.snowplowanalytics.snowplow", + name = "client_session", + format = "jsonschema", + version = "1-0-1"), + circeToJackson(json"""{ + "data": { + "userId": "some-fancy-user-session-id", + "sessionId": "42c8a55b-c0c2-4749-b9ac-09bb0d17d000", + "sessionIndex": 1, + "previousSessionId": null, + "storageMechanism": "COOKIE_1" + } + }""") ) - val configuration = parseJson( - """|{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "api_request_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "key": "user", - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "key": "userSession", - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", - | "jsonPath": "$.userId" - | } - | }, - | { - | "key": "client", - | "pojo": { - | "field": "app_id" - | } - | } - | ], - | "api": { - | "http": { - | "method": "GET", - | "uri": "http://api.acme.com/users/{{client}}/{{user}}?format=json", - | "timeout": 1000, - | "authentication": { - | "httpBasic": { - | "username": "xxx", - | "password": null - | } - | } - | } - | }, - | "outputs": [{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "json": { - | "jsonPath": "$.record" - | } - | }], - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | }""".stripMargin) + val configuration = parse( + """{ + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "api_request_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "key": "user", + "pojo": { + "field": "user_id" + } + }, + { + "key": "userSession", + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "jsonPath": "$.userId" + } + }, + { + "key": "client", + "pojo": { + "field": "app_id" + } + } + ], + "api": { + "http": { + "method": "GET", + "uri": "http://api.acme.com/users/{{client}}/{{user}}?format=json", + "timeout": 1000, + "authentication": { + "httpBasic": { + "username": "xxx", + "password": null + } + } + } + }, + "outputs": [{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "json": { + "jsonPath": "$.record" + } + }], + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get ApiRequestEnrichmentConfig.parse(configuration, SCHEMA_KEY) must beSuccessful(config) - val user = parseJson("""|{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "data": { - | "name": "Fancy User", - | "company": "Acme" - | } - |} - """.stripMargin) + val user = json"""{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "data": { + "name": "Fancy User", + "company": "Acme" + } + }""" apiSpy.perform( - url = "http://api.acme.com/users/some-fancy-app-id/some-fancy-user-id?format=json", + url = "http://api.acme.com/users/some-fancy-app-id/some-fancy-user-id?format=json", body = None ) returns """{"record": {"name": "Fancy User", "company": "Acme"}}""".success val enrichedContextResult = config.lookup( - event = fakeEnrichedEvent, + event = fakeEnrichedEvent, derivedContexts = List.empty, - customContexts = List(clientSession), - unstructEvent = List.empty + customContexts = List(clientSession), + unstructEvent = List.empty ) enrichedContextResult must beSuccessful(List(user)) } def e2 = { - val configuration = parseJson("""|{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "api_request_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "key": "user", - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "key": "user" - | }, - | { - | "key": "client", - | "pojo": { - | "field": "app_id" - | } - | } - | ], - | "api": { - | "http": { - | "method": "GET", - | "uri": "http://api.acme.com/users/{{client}}/{{user}}?format=json", - | "timeout": 1000, - | "authentication": { - | "httpBasic": { - | "username": "xxx", - | "password": "yyy" - | } - | } - | } - | }, - | "outputs": [{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "json": { - | "jsonPath": "$.record" - | } - | }], - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | }""".stripMargin) + val configuration = parse("""{ + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "api_request_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "key": "user", + "pojo": { + "field": "user_id" + } + }, + { + "key": "user" + }, + { + "key": "client", + "pojo": { + "field": "app_id" + } + } + ], + "api": { + "http": { + "method": "GET", + "uri": "http://api.acme.com/users/{{client}}/{{user}}?format=json", + "timeout": 1000, + "authentication": { + "httpBasic": { + "username": "xxx", + "password": "yyy" + } + } + } + }, + "outputs": [{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "json": { + "jsonPath": "$.record" + } + }], + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get ApiRequestEnrichmentConfig.parse(configuration, SCHEMA_KEY) must beFailing } def e3 = { - val configuration = parseJson( - """|{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "api_request_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "key": "user", - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "key": "client", - | "pojo": { - | "field": "app_id" - | }, - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", - | "jsonPath": "$.userId" - | } - | } - | ], - | "api": { - | "http": { - | "method": "GET", - | "uri": "http://api.acme.com/users/{{client}}/{{user}}?format=json", - | "timeout": 1000, - | "authentication": { - | "httpBasic": { - | "username": "xxx", - | "password": "yyy" - | } - | } - | } - | }, - | "outputs": [{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "json": { - | "jsonPath": "$.record" - | } - | }], - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | }""".stripMargin) + val configuration = parse( + """{ + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "api_request_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "key": "user", + "pojo": { + "field": "user_id" + } + }, + { + "key": "client", + "pojo": { + "field": "app_id" + }, + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "jsonPath": "$.userId" + } + } + ], + "api": { + "http": { + "method": "GET", + "uri": "http://api.acme.com/users/{{client}}/{{user}}?format=json", + "timeout": 1000, + "authentication": { + "httpBasic": { + "username": "xxx", + "password": "yyy" + } + } + } + }, + "outputs": [{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "json": { + "jsonPath": "$.record" + } + }], + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get ApiRequestEnrichmentConfig.parse(configuration, SCHEMA_KEY) must beFailing } @@ -268,107 +277,112 @@ class ApiRequestEnrichmentSpec extends Specification with ValidationMatchers wit val inputs = List( Input(key = "user", pojo = Some(PojoInput("user_id")), json = None), Input( - key = "userSession", + key = "userSession", pojo = None, json = Some( - JsonInput("contexts", "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", "$.userId"))), + JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "$.userId"))), Input(key = "client", pojo = Some(PojoInput("app_id")), json = None) ) - val api = HttpApi(method = "POST", - uri = "http://api.acme.com/users?format=json", - timeout = 1000, - authentication = Authentication(Some(HttpBasic(Some("xxx"), None)))) + val api = HttpApi( + method = "POST", + uri = "http://api.acme.com/users?format=json", + timeout = 1000, + authentication = Authentication(Some(HttpBasic(Some("xxx"), None)))) val apiSpy = spy(api) - val output = Output(schema = "iglu:com.acme/user/jsonschema/1-0-0", json = Some(JsonOutput("$.record"))) - val cache = Cache(size = 3000, ttl = 60) + val output = + Output(schema = "iglu:com.acme/user/jsonschema/1-0-0", json = Some(JsonOutput("$.record"))) + val cache = Cache(size = 3000, ttl = 60) val config = ApiRequestEnrichment(inputs, apiSpy, List(output), cache) val fakeEnrichedEvent = new EnrichedEvent { - app_id = "some-fancy-app-id" + app_id = "some-fancy-app-id" user_id = "some-fancy-user-id" } val clientSession: JsonSchemaPair = ( - SchemaKey(vendor = "com.snowplowanalytics.snowplow", - name = "client_session", - format = "jsonschema", - version = "1-0-1"), - JsonMethods.asJsonNode(parseJson("""|{ - | "data": { - | "userId": "some-fancy-user-session-id", - | "sessionId": "42c8a55b-c0c2-4749-b9ac-09bb0d17d000", - | "sessionIndex": 1, - | "previousSessionId": null, - | "storageMechanism": "COOKIE_1" - | } - |}""".stripMargin)) + SchemaKey( + vendor = "com.snowplowanalytics.snowplow", + name = "client_session", + format = "jsonschema", + version = "1-0-1"), + circeToJackson(json"""{ + "data": { + "userId": "some-fancy-user-session-id", + "sessionId": "42c8a55b-c0c2-4749-b9ac-09bb0d17d000", + "sessionIndex": 1, + "previousSessionId": null, + "storageMechanism": "COOKIE_1" + } + }""") ) - val configuration = parseJson( - """|{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "api_request_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "key": "user", - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "key": "userSession", - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", - | "jsonPath": "$.userId" - | } - | }, - | { - | "key": "client", - | "pojo": { - | "field": "app_id" - | } - | } - | ], - | "api": { - | "http": { - | "method": "POST", - | "uri": "http://api.acme.com/users?format=json", - | "timeout": 1000, - | "authentication": { - | "httpBasic": { - | "username": "xxx", - | "password": null - | } - | } - | } - | }, - | "outputs": [{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "json": { - | "jsonPath": "$.record" - | } - | }], - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | }""".stripMargin) + val configuration = parse( + """{ + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "api_request_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "key": "user", + "pojo": { + "field": "user_id" + } + }, + { + "key": "userSession", + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "jsonPath": "$.userId" + } + }, + { + "key": "client", + "pojo": { + "field": "app_id" + } + } + ], + "api": { + "http": { + "method": "POST", + "uri": "http://api.acme.com/users?format=json", + "timeout": 1000, + "authentication": { + "httpBasic": { + "username": "xxx", + "password": null + } + } + } + }, + "outputs": [{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "json": { + "jsonPath": "$.record" + } + }], + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get ApiRequestEnrichmentConfig.parse(configuration, SCHEMA_KEY) must beSuccessful(config) - val user = parseJson("""|{ - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "data": { - | "name": "Fancy User", - | "company": "Acme" - | } - |} - """.stripMargin) + val user = json"""{ + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "data": { + "name": "Fancy User", + "company": "Acme" + } + }""" apiSpy.perform( url = "http://api.acme.com/users?format=json", @@ -377,10 +391,10 @@ class ApiRequestEnrichmentSpec extends Specification with ValidationMatchers wit ) returns """{"record": {"name": "Fancy User", "company": "Acme"}}""".success val enrichedContextResult = config.lookup( - event = fakeEnrichedEvent, + event = fakeEnrichedEvent, derivedContexts = List.empty, - customContexts = List(clientSession), - unstructEvent = List.empty + customContexts = List(clientSession), + unstructEvent = List.empty ) enrichedContextResult must beSuccessful(List(user)) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/CacheSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/CacheSpec.scala index 457e78371..e024f02cc 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/CacheSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/CacheSpec.scala @@ -12,7 +12,7 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest -import org.json4s.JInt +import io.circe._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers import org.specs2.mock.Mockito @@ -30,45 +30,45 @@ class CacheSpec extends Specification with ValidationMatchers with Mockito { def e1 = { val cache = Cache(3, 2) - val key = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url", body = None) + val key = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url", body = None) - cache.put(key, JInt(42).success) - cache.put(key, JInt(52).success) + cache.put(key, Json.fromInt(42).success) + cache.put(key, Json.fromInt(52).success) cache.get(key) must beSome.like { - case v => v must beSuccessful(JInt(52)) + case v => v must beSuccessful(Json.fromInt(52)) } and (cache.actualLoad must beEqualTo(1)) } def e2 = { val cache = Cache(3, 2) - val key = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url", body = None) - cache.put(key, JInt(42).success) + val key = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url", body = None) + cache.put(key, Json.fromInt(42).success) Thread.sleep(3000) cache.get(key) must beNone and (cache.actualLoad must beEqualTo(0)) } def e3 = { val cache = Cache(2, 2) - val key1 = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url1", body = None) - cache.put(key1, JInt(32).success) + val key1 = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url1", body = None) + cache.put(key1, Json.fromInt(32).success) val key2 = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url2", body = None) - cache.put(key2, JInt(32).success) + cache.put(key2, Json.fromInt(32).success) val key3 = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url3", body = None) - cache.put(key3, JInt(32).success) + cache.put(key3, Json.fromInt(32).success) cache.get(key1) must beNone and (cache.actualLoad must beEqualTo(2)) } def e4 = { val cache = Cache(3, 2) - val key = ApiRequestEnrichment.cacheKey(url = "http://api.acme.com/url", body = Some("""{"value":"42"}""")) + val key = ApiRequestEnrichment.cacheKey("http://api.acme.com/url", Some("""{"value":"42"}""")) - cache.put(key, JInt(33).success) - cache.put(key, JInt(42).success) + cache.put(key, Json.fromInt(33).success) + cache.put(key, Json.fromInt(42).success) cache.get(key) must beSome.like { - case v => v must beSuccessful(JInt(42)) + case v => v must beSuccessful(Json.fromInt(42)) } and (cache.actualLoad must beEqualTo(1)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala index c0652710b..b525a8024 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala @@ -26,21 +26,23 @@ class HttpApiSpec extends Specification with ValidationMatchers with Mockito { """ def e1 = { - val httpApi = HttpApi("GET", "http://api.acme.com/{{user}}/{{ time}}/", anyInt, Authentication(None)) + val httpApi = + HttpApi("GET", "http://api.acme.com/{{user}}/{{ time}}/", anyInt, Authentication(None)) val templateContext = Map("user" -> "admin") - val request = httpApi.buildUrl(templateContext) + val request = httpApi.buildUrl(templateContext) request must beNone } def e2 = { val httpApi = - HttpApi(anyString, - "http://thishostdoesntexist31337:8123/{{ user }}/foo/{{ time}}/{{user}}", - anyInt, - Authentication(None)) + HttpApi( + anyString, + "http://thishostdoesntexist31337:8123/{{ user }}/foo/{{ time}}/{{user}}", + anyInt, + Authentication(None)) val templateContext = Map("user" -> "admin", "time" -> "November 2015") - val request = httpApi.buildUrl(templateContext) + val request = httpApi.buildUrl(templateContext) request must beSome("http://thishostdoesntexist31337:8123/admin/foo/November+2015/admin") } @@ -52,7 +54,7 @@ class HttpApiSpec extends Specification with ValidationMatchers with Mockito { List(Output("", Some(JsonOutput("")))), Cache(1, 1)) - val event = new outputs.EnrichedEvent + val event = new outputs.EnrichedEvent val request = enrichment.lookup(event, Nil, Nil, Nil) request must beFailing } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala index 767ff2502..32359a80a 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala @@ -13,8 +13,8 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry.apirequest -import org.json4s.JObject -import org.json4s.jackson.parseJson +import io.circe._ +import io.circe.literal._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers import scalaz._ @@ -36,83 +36,83 @@ class InputSpec extends Specification with ValidationMatchers { """ object ContextCase { - val ccInput = - Input("nullableValue", - pojo = None, - json = JsonInput("contexts", "iglu:org.ietf/http_cookie/jsonschema/1-*-*", "$.value").some) + val ccInput = Input( + "nullableValue", + pojo = None, + json = JsonInput("contexts", "iglu:org.ietf/http_cookie/jsonschema/1-*-*", "$.value").some + ) val derInput = Input( "overridenValue", pojo = None, - json = JsonInput("derived_contexts", "iglu:org.openweathermap/weather/jsonschema/1-0-*", "$.main.humidity").some) + json = JsonInput( + "derived_contexts", + "iglu:org.openweathermap/weather/jsonschema/1-0-*", + "$.main.humidity" + ).some + ) val unstructInput = Input( "unstructValue", pojo = None, - json = JsonInput("unstruct_event", - "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", - "$.state").some + json = JsonInput( + "unstruct_event", + "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", + "$.state" + ).some ) val overrideHumidityInput = Input( "overridenValue", pojo = None, - json = JsonInput("contexts", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "$.latitude").some) - - val derivedContext1 = parseJson( - """ - |{ - | "schema": "iglu:org.openweathermap/weather/jsonschema/1-0-0", - | "data": { - | "clouds": { - | "all": 0 - | }, - | "dt": "2014-11-10T08:38:30.000Z", - | "main": { - | "grnd_level": 1021.91, - | "humidity": 90, - | "pressure": 1021.91, - | "sea_level": 1024.77, - | "temp": 301.308, - | "temp_max": 301.308, - | "temp_min": 301.308 - | }, - | "weather": [ { "description": "Sky is Clear", "icon": "01d", "id": 800, "main": "Clear" } ], - | "wind": { - | "deg": 190.002, - | "speed": 4.39 - | } - |} - |} - """.stripMargin).asInstanceOf[JObject] - - val cookieContext = parseJson(""" - |{ - | "schema": "iglu:org.ietf/http_cookie/jsonschema/1-0-0", - | "data": {"name": "someCookieAgain", "value": null} - |} - """.stripMargin).asInstanceOf[JObject] - - val unstructEvent = parseJson( - """ - |{ - | "schema": "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", - | "data": {"name": "Some EMR Job", "state": "COMPLETED"} - |} - """.stripMargin).asInstanceOf[JObject] - - val overriderContext = parseJson( - """ - |{ - | "schema": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0", - | "data": {"latitude": 43.1, "longitude": 32.1} - |} - """.stripMargin).asInstanceOf[JObject] + json = JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "$.latitude" + ).some + ) + + val derivedContext1 = json"""{ + "schema": "iglu:org.openweathermap/weather/jsonschema/1-0-0", + "data": { + "clouds": { + "all": 0 + }, + "dt": "2014-11-10T08:38:30.000Z", + "main": { + "grnd_level": 1021.91, + "humidity": 90, + "pressure": 1021.91, + "sea_level": 1024.77, + "temp": 301.308, + "temp_max": 301.308, + "temp_min": 301.308 + }, + "weather": [ { "description": "Sky is Clear", "icon": "01d", "id": 800, "main": "Clear" } ], + "wind": { + "deg": 190.002, + "speed": 4.39 + } + } + }""" + + val cookieContext = json"""{ + "schema": "iglu:org.ietf/http_cookie/jsonschema/1-0-0", + "data": {"name": "someCookieAgain", "value": null} + }""" + + val unstructEvent = json"""{ + "schema": "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", + "data": {"name": "Some EMR Job", "state": "COMPLETED"} + }""" + + val overriderContext = json"""{ + "schema": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0", + "data": {"latitude": 43.1, "longitude": 32.1} + }""" } def e1 = { val input1 = Input("user", pojo = PojoInput("user_id").some, json = None) val input2 = Input("time", pojo = PojoInput("true_tstamp").some, json = None) - val event = new EnrichedEvent + val event = new EnrichedEvent event.setUser_id("chuwy") event.setTrue_tstamp("20") val templateContext = Input.buildTemplateContext(List(input1, input2), event, Nil, Nil, None) @@ -126,36 +126,36 @@ class InputSpec extends Specification with ValidationMatchers { List(ccInput, derInput, unstructInput, overrideHumidityInput), event, derivedContexts = List(derivedContext1), - customContexts = List(cookieContext, overriderContext), - unstructEvent = Some(unstructEvent) + customContexts = List(cookieContext, overriderContext), + unstructEvent = Some(unstructEvent) ) templateContext must beSuccessful( - Some(Map("nullableValue" -> "null", "overridenValue" -> "43.1", "unstructValue" -> "COMPLETED"))) + Some( + Map("nullableValue" -> "null", "overridenValue" -> "43.1", "unstructValue" -> "COMPLETED"))) } def e3 = { val jsonLatitudeInput = Input( "latitude", pojo = None, - json = JsonInput("contexts", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "$.latitude").some) + json = JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "$.latitude").some) val pojoLatitudeInput = Input("latitude", pojo = PojoInput("geo_latitude").some, json = None) - val jsonLatitudeContext = parseJson( - """ - |{ - | "schema": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0", - | "data": {"latitude": 43.1, "longitude": 32.1} - |} - """.stripMargin).asInstanceOf[JObject] + val jsonLatitudeContext = json"""{ + "schema": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0", + "data": {"latitude": 43.1, "longitude": 32.1} + }""" val event = new EnrichedEvent event.setGeo_latitude(42.0f) - val templateContext = Input.buildTemplateContext(List(jsonLatitudeInput, pojoLatitudeInput), - event, - derivedContexts = Nil, - customContexts = List(jsonLatitudeContext), - unstructEvent = None) + val templateContext = Input.buildTemplateContext( + List(jsonLatitudeInput, pojoLatitudeInput), + event, + derivedContexts = Nil, + customContexts = List(jsonLatitudeContext), + unstructEvent = None) templateContext must beSuccessful(Some(Map("latitude" -> "43.1"))) } @@ -163,31 +163,35 @@ class InputSpec extends Specification with ValidationMatchers { val invalidJsonPathInput = Input( "latitude", pojo = None, - json = JsonInput("contexts", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "*.invalidJsonPath").some) + json = JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "*.invalidJsonPath").some) val invalidJsonFieldInput = Input( "latitude", pojo = None, - json = JsonInput("invalid_field", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "$.validJsonPath").some + json = JsonInput( + "invalid_field", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "$.validJsonPath").some ) val pojoInput = Input("latitude", pojo = PojoInput("app_id").some, json = None) - val templateContext = Input.buildTemplateContext(List(invalidJsonPathInput, pojoInput, invalidJsonFieldInput), - null, - derivedContexts = Nil, - customContexts = List(JObject(Nil)), - unstructEvent = None) + val templateContext = Input.buildTemplateContext( + List(invalidJsonPathInput, pojoInput, invalidJsonFieldInput), + null, + derivedContexts = Nil, + customContexts = List(Json.fromJsonObject(JsonObject.empty)), + unstructEvent = None + ) templateContext must beFailing.like { case errors => errors.toList must have length (3) } } def e5 = { - val event = new EnrichedEvent - val pojoInput = Input("someKey", pojo = PojoInput("app_id").some, json = None) + val event = new EnrichedEvent + val pojoInput = Input("someKey", pojo = PojoInput("app_id").some, json = None) val templateContext: ValidationNel[String, Option[Any]] = pojoInput.getFromEvent(event) templateContext must beSuccessful.like { case map => map must beNone @@ -195,20 +199,21 @@ class InputSpec extends Specification with ValidationMatchers { } def e6 = { - val event = new EnrichedEvent - val pojoInput = Input("someKey", pojo = PojoInput("unknown_property").some, json = None) + val event = new EnrichedEvent + val pojoInput = Input("someKey", pojo = PojoInput("unknown_property").some, json = None) val templateContext: ValidationNel[String, Option[Any]] = pojoInput.getFromEvent(event) templateContext must beFailing } def e7 = { - val input1 = Input("user", Some(PojoInput("user_id")), None) - val input2 = Input("time", Some(PojoInput("true_tstamp")), None) + val input1 = Input("user", Some(PojoInput("user_id")), None) + val input2 = Input("time", Some(PojoInput("true_tstamp")), None) val uriTemplate = "http://thishostdoesntexist31337:8123/{{ user }}/foo/{{ time}}/{{user}}" - val enrichment = ApiRequestEnrichment(List(input1, input2), - HttpApi("GET", uriTemplate, 1000, Authentication(None)), - List(Output("iglu:someschema", JsonOutput("$").some)), - Cache(10, 5)) + val enrichment = ApiRequestEnrichment( + List(input1, input2), + HttpApi("GET", uriTemplate, 1000, Authentication(None)), + List(Output("iglu:someschema", JsonOutput("$").some)), + Cache(10, 5)) val event = new outputs.EnrichedEvent event.setUser_id("chuwy") // time in true_tstamp won't be found @@ -219,19 +224,20 @@ class InputSpec extends Specification with ValidationMatchers { } def e8 = { - val input = - Input("permissive", - None, - Some(JsonInput("contexts", "iglu:com.snowplowanalytics/some_schema/jsonschema/*-*-*", "$.somekey"))) - - val obj: JObject = parseJson(""" - |{ - | "schema": "iglu:com.snowplowanalytics/some_schema/jsonschema/2-0-1", - | "data": { - | "somekey": "somevalue" - | } - | } - """.stripMargin).asInstanceOf[JObject] + val input = Input( + "permissive", + None, + Some( + JsonInput( + "contexts", + "iglu:com.snowplowanalytics/some_schema/jsonschema/*-*-*", + "$.somekey")) + ) + + val obj = json"""{ + "schema": "iglu:com.snowplowanalytics/some_schema/jsonschema/2-0-1", + "data": { "somekey": "somevalue" } + }""" input.getFromJson(Nil, List(obj), None) must beSuccessful.like { case option => diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala index 8f70ae0d7..61502c057 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala @@ -12,8 +12,8 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.apirequest -import org.json4s.JObject -import org.json4s.JsonDSL._ +import io.circe._ +import io.circe.literal._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers @@ -26,22 +26,33 @@ class OutputSpec extends Specification with ValidationMatchers { """ def e1 = { - val output = Output("iglu:com.snowplowanalytics/some_schema/jsonschema/1-0-0", Some(JsonOutput("$.value"))) - output.extract(JObject(Nil)) must beFailing + val output = + Output("iglu:com.snowplowanalytics/some_schema/jsonschema/1-0-0", Some(JsonOutput("$.value"))) + output.extract(Json.fromJsonObject(JsonObject.empty)) must beFailing } def e2 = { - val output = Output("iglu:com.snowplowanalytics/some_schema/jsonschema/1-0-0", Some(JsonOutput("$.value"))) - output.parse("""{"value": 32}""").flatMap(output.extract).map(output.describeJson) must beSuccessful.like { + val output = Output( + "iglu:com.snowplowanalytics/some_schema/jsonschema/1-0-0", + Some(JsonOutput("$.value")) + ) + output + .parseResponse("""{"value": 32}""") + .flatMap(output.extract) + .map(output.describeJson) must beSuccessful.like { case context => - context must be equalTo (("schema", "iglu:com.snowplowanalytics/some_schema/jsonschema/1-0-0") ~ ("data" -> 32)) + context must be equalTo json"""{ + "schema": "iglu:com.snowplowanalytics/some_schema/jsonschema/1-0-0", + "data": 32 + }""" } } def e3 = { - val output = Output("iglu:com.snowplowanalytics/complex_schema/jsonschema/1-0-0", - Some(JsonOutput("$.objects[1].deepNesting[3]"))) - output.parse(""" + val output = Output( + "iglu:com.snowplowanalytics/complex_schema/jsonschema/1-0-0", + Some(JsonOutput("$.objects[1].deepNesting[3]"))) + output.parseResponse(""" |{ | "value": 32, | "objects": @@ -53,8 +64,10 @@ class OutputSpec extends Specification with ValidationMatchers { |} """.stripMargin).flatMap(output.extract).map(output.describeJson) must beSuccessful.like { case context => - context must be equalTo (("schema", "iglu:com.snowplowanalytics/complex_schema/jsonschema/1-0-0") ~ ("data" -> 42)) - + context must be equalTo json"""{ + "schema": "iglu:com.snowplowanalytics/complex_schema/jsonschema/1-0-0", + "data": 42 + }""" } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala index fa28159c0..b718a4c2b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala @@ -17,20 +17,20 @@ package pii import java.net.URI +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{Resolver, SchemaCriterion} import com.snowplowanalytics.iglu.client.repositories.RepositoryRefConfig +import io.circe.parser._ import org.joda.time.DateTime -import org.json4s.jackson.JsonMethods.parse import org.apache.commons.codec.digest.DigestUtils import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers import scalaz._ import Scalaz._ +import SpecHelpers.toNameValuePairs import loaders.{CollectorApi, CollectorContext, CollectorPayload, CollectorSource} import outputs.EnrichedEvent -import utils.{ScalazJson4sUtils, TestResourcesRepositoryRef} -import SpecHelpers.toNameValuePairs import utils.TestResourcesRepositoryRef class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatchers { @@ -49,16 +49,22 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche def commonSetup(enrichmentMap: EnrichmentMap): List[ValidatedEnrichedEvent] = { val enrichmentRegistry = EnrichmentRegistry(enrichmentMap) val context = - CollectorContext(Some(DateTime.parse("2017-07-14T03:39:39.000+00:00")), Some("127.0.0.1"), None, None, Nil, None) + CollectorContext( + Some(DateTime.parse("2017-07-14T03:39:39.000+00:00")), + Some("127.0.0.1"), + None, + None, + Nil, + None) val source = CollectorSource("clj-tomcat", "UTF-8", None) val collectorPayload = CollectorPayload( CollectorApi("com.snowplowanalytics.snowplow", "tp2"), toNameValuePairs( - "e" -> "se", + "e" -> "se", "aid" -> "ads", "uid" -> "john@acme.com", - "ip" -> "70.46.123.145", - "fp" -> "its_you_again!", + "ip" -> "70.46.123.145", + "fp" -> "its_you_again!", "co" -> """ |{ @@ -120,18 +126,18 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche context ) val input: ValidatedMaybeCollectorPayload = Some(collectorPayload).successNel - val rrc = new RepositoryRefConfig("test-schema", 1, List("com.snowplowanalytics.snowplow")) - val repos = TestResourcesRepositoryRef(rrc, "src/test/resources/iglu-schemas") - val adapterRegistry = new AdapterRegistry() - implicit val resolver = new Resolver(repos = List(repos)) - EtlPipeline.processEvents(adapterRegistry, enrichmentRegistry, s"spark-0.0.0", new DateTime(1500000000L), input) + val rrc = new RepositoryRefConfig("test-schema", 1, List("com.snowplowanalytics.snowplow")) + val repos = TestResourcesRepositoryRef(rrc, "src/test/resources/iglu-schemas") + implicit val resolver = new Resolver(repos = List(repos)) + EtlPipeline.processEvents(registry, s"spark-0.0.0", new DateTime(1500000000L), input) } - private val ipEnrichment = IpLookupsEnrichment(Some(("geo", new URI("/ignored-in-local-mode/"), "GeoIP2-City.mmdb")), - Some(("isp", new URI("/ignored-in-local-mode/"), "GeoIP2-ISP.mmdb")), - None, - None, - true) + private val ipEnrichment = IpLookupsEnrichment( + Some(("geo", new URI("/ignored-in-local-mode/"), "GeoIP2-City.mmdb")), + Some(("isp", new URI("/ignored-in-local-mode/"), "GeoIP2-ISP.mmdb")), + None, + None, + true) def e1 = { val enrichmentMap = Map( @@ -148,18 +154,21 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche ) ), false, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") ) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.user_id = "7d8a4beae5bc9d314600667d2f410918f9af265017a6ade99f60a9c8f3aac6e9" - expected.user_ipaddress = "dd9720903c89ae891ed5c74bb7a9f2f90f6487927ac99afe73b096ad0287f3f5" - expected.ip_domain = null + expected.app_id = "ads" + expected.user_id = "7d8a4beae5bc9d314600667d2f410918f9af265017a6ade99f60a9c8f3aac6e9" + expected.user_ipaddress = "dd9720903c89ae891ed5c74bb7a9f2f90f6487927ac99afe73b096ad0287f3f5" + expected.ip_domain = null expected.user_fingerprint = "27abac60dff12792c6088b8d00ce7f25c86b396b8c3740480cd18e21068ecff4" - expected.geo_city = null - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.geo_city = null + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" output.size must_== 1 val out = output.head @@ -182,59 +191,81 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche "pii_enrichment_config" -> PiiPseudonymizerEnrichment( List( PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-*").toOption.get, - jsonPath = "$.emailAddress" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-*").toOption.get, + jsonPath = "$.emailAddress" ), PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-1-0").toOption.get, - jsonPath = "$.data.emailAddress2" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-1-0").toOption.get, + jsonPath = "$.data.emailAddress2" ), PiiJson( - fieldMutator = JsonMutators.get("unstruct_event").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.mailgun/message_clicked/jsonschema/1-0-0").toOption.get, - jsonPath = "$.ip" + fieldMutator = JsonMutators.get("unstruct_event").get, + schemaCriterion = SchemaCriterion + .parse("iglu:com.mailgun/message_clicked/jsonschema/1-0-0") + .toOption + .get, + jsonPath = "$.ip" ) ), false, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") ) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.user_id = "john@acme.com" - expected.user_ipaddress = "70.46.123.145" - expected.ip_domain = null + expected.app_id = "ads" + expected.user_id = "john@acme.com" + expected.user_ipaddress = "70.46.123.145" + expected.ip_domain = null expected.user_fingerprint = "its_you_again!" - expected.geo_city = null - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.geo_city = null + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" output.size must_== 1 val out = output.head out must beSuccessful.like { case enrichedEvent => - implicit val formats = org.json4s.DefaultFormats - val contextJ = parse(enrichedEvent.contexts) - val unstructEventJ = parse(enrichedEvent.unstruct_event) - (((contextJ \ "data")(0) \ "data" \ "emailAddress") - .extract[String] must_== "72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") and - (((contextJ \ "data")(0) \ "data" \ "emailAddress2").extract[String] must_== "bob@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress").extract[String] must_== "tim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress2").extract[String] must_== "tom@acme.com") and - // The following three tests are for the case that the context schema allows the fields data and schema - // and in addition the schema field matches the configured schema. There should be no replacement there - // (unless that is specified in jsonpath) - (((contextJ \ "data")(1) \ "data" \ "data" \ "emailAddress").extract[String] must_== "jim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "data" \ "emailAddress2") - .extract[String] must_== "1c6660411341411d5431669699149283d10e070224be4339d52bbc4b007e78c5") and - (((contextJ \ "data")(1) \ "data" \ "schema") - .extract[String] must_== "iglu:com.acme/email_sent/jsonschema/1-0-0") and - (((unstructEventJ \ "data") \ "data" \ "ip") - .extract[String] must_== "269c433d0cc00395e3bc5fe7f06c5ad822096a38bec2d8a005367b52c0dfb428") and - (((unstructEventJ \ "data") \ "data" \ "myVar2").extract[String] must_== "awesome") + val contextJ = parse(enrichedEvent.contexts).toOption.get.hcursor + val contextJFirstElement = contextJ.downField("data").downArray + val contextJSecondElement = contextJFirstElement.right + val unstructEventJ = parse(enrichedEvent.unstruct_event).toOption.get.hcursor + .downField("data") + .downField("data") + + contextJFirstElement.downField("data").get[String]("emailAddress") must_== + Right("72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") + contextJFirstElement.downField("data").get[String]("emailAddress2") must_== + Right("bob@acme.com") + contextJSecondElement.downField("data").get[String]("emailAddress") must_== + Right("tim@acme.com") + contextJSecondElement.downField("data").get[String]("emailAddress2") must_== + Right("tom@acme.com") + // The following three tests are for the case that the context schema allows the fields + // data and schema and in addition the schema field matches the configured schema. There + // should be no replacement there (unless that is specified in jsonpath) + contextJSecondElement + .downField("data") + .downField("data") + .get[String]("emailAddress") must_== Right("jim@acme.com") + contextJSecondElement + .downField("data") + .downField("data") + .get[String]("emailAddress2") must_== + Right("1c6660411341411d5431669699149283d10e070224be4339d52bbc4b007e78c5") + contextJSecondElement.downField("data").get[String]("schema") must_== + Right("iglu:com.acme/email_sent/jsonschema/1-0-0") + + unstructEventJ.get[String]("ip") must_== + Right("269c433d0cc00395e3bc5fe7f06c5ad822096a38bec2d8a005367b52c0dfb428") + unstructEventJ.get[String]("myVar2") must_== Right("awesome") } } @@ -244,36 +275,41 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche "pii_enrichment_config" -> PiiPseudonymizerEnrichment( List( PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-*").toOption.get, - jsonPath = "$.field.that.does.not.exist.in.this.instance" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-*").toOption.get, + jsonPath = "$.field.that.does.not.exist.in.this.instance" ) ), false, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") ) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.user_id = "john@acme.com" - expected.user_ipaddress = "70.46.123.145" - expected.ip_domain = null + expected.app_id = "ads" + expected.user_id = "john@acme.com" + expected.user_ipaddress = "70.46.123.145" + expected.ip_domain = null expected.user_fingerprint = "its_you_again!" - expected.geo_city = null - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.geo_city = null + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" output.size must_== 1 val out = output.head out must beSuccessful.like { case enrichedEvent => - implicit val formats = org.json4s.DefaultFormats - val contextJ = parse(enrichedEvent.contexts) - (((contextJ \ "data")(0) \ "data" \ "emailAddress").extract[String] must_== "jim@acme.com") and - (((contextJ \ "data")(0) \ "data" \ "emailAddress2").extract[String] must_== "bob@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress").extract[String] must_== "tim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress2").extract[String] must_== "tom@acme.com") + val contextJ = parse(enrichedEvent.contexts).toOption.get.hcursor.downField("data") + val firstElem = contextJ.downArray.downField("data") + val secondElem = contextJ.downArray.right.downField("data") + firstElem.get[String]("emailAddress") must_== Right("jim@acme.com") + firstElem.get[String]("emailAddress2") must_== Right("bob@acme.com") + secondElem.get[String]("emailAddress") must_== Right("tim@acme.com") + secondElem.get[String]("emailAddress2") must_== Right("tom@acme.com") } } @@ -283,38 +319,43 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche "pii_enrichment_config" -> PiiPseudonymizerEnrichment( List( PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-*").toOption.get, - jsonPath = "$.['emailAddress', 'emailAddress2', 'emailAddressNonExistent']" // Last case throws an exeption if misconfigured + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-*").toOption.get, + jsonPath = "$.['emailAddress', 'emailAddress2', 'emailAddressNonExistent']" // Last case throws an exeption if misconfigured ) ), false, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") ) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.user_id = "john@acme.com" - expected.user_ipaddress = "70.46.123.145" - expected.ip_domain = null + expected.app_id = "ads" + expected.user_id = "john@acme.com" + expected.user_ipaddress = "70.46.123.145" + expected.ip_domain = null expected.user_fingerprint = "its_you_again!" - expected.geo_city = null - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.geo_city = null + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" output.size must_== 1 val out = output.head out must beSuccessful.like { case enrichedEvent => - implicit val formats = org.json4s.DefaultFormats - val contextJ = parse(enrichedEvent.contexts) - (((contextJ \ "data")(0) \ "data" \ "emailAddress") - .extract[String] must_== "72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") and - (((contextJ \ "data")(0) \ "data" \ "emailAddress2") - .extract[String] must_== "1c6660411341411d5431669699149283d10e070224be4339d52bbc4b007e78c5") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress").extract[String] must_== "tim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress2").extract[String] must_== "tom@acme.com") + val contextJ = parse(enrichedEvent.contexts).toOption.get.hcursor.downField("data") + val firstElem = contextJ.downArray.downField("data") + val secondElem = contextJ.downArray.right.downField("data") + firstElem.get[String]("emailAddress") must_== + Right("72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") + firstElem.get[String]("emailAddress2") must_== + Right("1c6660411341411d5431669699149283d10e070224be4339d52bbc4b007e78c5") + secondElem.get[String]("emailAddress") must_== Right("tim@acme.com") + secondElem.get[String]("emailAddress2") must_== Right("tom@acme.com") } } @@ -324,38 +365,42 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche "pii_enrichment_config" -> PiiPseudonymizerEnrichment( List( PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-0").toOption.get, - jsonPath = "$.emailAddress" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-0").toOption.get, + jsonPath = "$.emailAddress" ) ), false, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") ) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.user_id = "john@acme.com" - expected.user_ipaddress = "70.46.123.145" - expected.ip_domain = null + expected.app_id = "ads" + expected.user_id = "john@acme.com" + expected.user_ipaddress = "70.46.123.145" + expected.ip_domain = null expected.user_fingerprint = "its_you_again!" - expected.geo_city = null - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.geo_city = null + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" output.size must_== 1 val out = output.head out must beSuccessful.like { case enrichedEvent => - implicit val formats = org.json4s.DefaultFormats - val contextJ = parse(enrichedEvent.contexts) - (((contextJ \ "data")(0) \ "data" \ "emailAddress") - .extract[String] must_== "72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") and - (((contextJ \ "data")(0) \ "data" \ "emailAddress2") - .extract[String] must_== "bob@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress") - .extract[String] must_== "09e4160b10703767dcb28d834c1905a182af0f828d6d3512dd07d466c283c840") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress2").extract[String] must_== "tom@acme.com") + val contextJ = parse(enrichedEvent.contexts).toOption.get.hcursor.downField("data") + val firstElem = contextJ.downArray.downField("data") + val secondElem = contextJ.downArray.right.downField("data") + firstElem.get[String]("emailAddress") must_== + Right("72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") + firstElem.get[String]("emailAddress2") must_== Right("bob@acme.com") + secondElem.get[String]("emailAddress") must_== + Right("09e4160b10703767dcb28d834c1905a182af0f828d6d3512dd07d466c283c840") + secondElem.get[String]("emailAddress2") must_== Right("tom@acme.com") } } @@ -365,39 +410,41 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche "pii_enrichment_config" -> PiiPseudonymizerEnrichment( List( PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-*").toOption.get, - jsonPath = "$.someInt" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-*").toOption.get, + jsonPath = "$.someInt" ) ), false, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") ) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.user_id = "john@acme.com" - expected.user_ipaddress = "70.46.123.145" - expected.ip_domain = null + expected.app_id = "ads" + expected.user_id = "john@acme.com" + expected.user_ipaddress = "70.46.123.145" + expected.ip_domain = null expected.user_fingerprint = "its_you_again!" - expected.geo_city = null - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.geo_city = null + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" output.size must_== 1 val out = output.head out must beSuccessful.like { case enrichedEvent => - implicit val formats = org.json4s.DefaultFormats - val contextJ = parse(enrichedEvent.contexts) - (((contextJ \ "data")(0) \ "data" \ "emailAddress") - .extract[String] must_== "jim@acme.com") and - (((contextJ \ "data")(0) \ "data" \ "emailAddress2") - .extract[String] must_== "bob@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress") - .extract[String] must_== "tim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress2").extract[String] must_== "tom@acme.com") - (((contextJ \ "data")(1) \ "data" \ "someInt").extract[Int] must_== 1) + val contextJ = parse(enrichedEvent.contexts).toOption.get.hcursor.downField("data") + val firstElem = contextJ.downArray.downField("data") + val secondElem = contextJ.downArray.right.downField("data") + firstElem.get[String]("emailAddress") must_== Right("jim@acme.com") + firstElem.get[String]("emailAddress2") must_== Right("bob@acme.com") + secondElem.get[String]("emailAddress") must_== Right("im@acme.com") + secondElem.get[String]("emailAddress2") must_== Right("tom@acme.com") + secondElem.get[Int]("someInt") must_== Right(1) } } @@ -411,31 +458,39 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche PiiScalar(fieldMutator = ScalarMutators.get("ip_domain").get), PiiScalar(fieldMutator = ScalarMutators.get("user_fingerprint").get), PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-*").toOption.get, - jsonPath = "$.emailAddress" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-*").toOption.get, + jsonPath = "$.emailAddress" ), PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-1-0").toOption.get, - jsonPath = "$.data.emailAddress2" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-1-0").toOption.get, + jsonPath = "$.data.emailAddress2" ), PiiJson( - fieldMutator = JsonMutators.get("unstruct_event").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.mailgun/message_clicked/jsonschema/1-0-0").toOption.get, - jsonPath = "$.ip" + fieldMutator = JsonMutators.get("unstruct_event").get, + schemaCriterion = SchemaCriterion + .parse("iglu:com.mailgun/message_clicked/jsonschema/1-0-0") + .toOption + .get, + jsonPath = "$.ip" ) ), true, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") ) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.ip_domain = null - expected.geo_city = null - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.app_id = "ads" + expected.ip_domain = null + expected.geo_city = null + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" expected.pii = """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/pii_transformation/jsonschema/1-0-0","data":{"pii":{"pojo":[{"fieldName":"user_fingerprint","originalValue":"its_you_again!","modifiedValue":"27abac60dff12792c6088b8d00ce7f25c86b396b8c3740480cd18e21068ecff4"},{"fieldName":"user_ipaddress","originalValue":"70.46.123.145","modifiedValue":"dd9720903c89ae891ed5c74bb7a9f2f90f6487927ac99afe73b096ad0287f3f5"},{"fieldName":"user_id","originalValue":"john@acme.com","modifiedValue":"7d8a4beae5bc9d314600667d2f410918f9af265017a6ade99f60a9c8f3aac6e9"}],"json":[{"fieldName":"unstruct_event","originalValue":"50.56.129.169","modifiedValue":"269c433d0cc00395e3bc5fe7f06c5ad822096a38bec2d8a005367b52c0dfb428","jsonPath":"$.ip","schema":"iglu:com.mailgun/message_clicked/jsonschema/1-0-0"},{"fieldName":"contexts","originalValue":"bob@acme.com","modifiedValue":"1c6660411341411d5431669699149283d10e070224be4339d52bbc4b007e78c5","jsonPath":"$.data.emailAddress2","schema":"iglu:com.acme/email_sent/jsonschema/1-1-0"},{"fieldName":"contexts","originalValue":"jim@acme.com","modifiedValue":"72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6","jsonPath":"$.emailAddress","schema":"iglu:com.acme/email_sent/jsonschema/1-0-0"}]},"strategy":{"pseudonymize":{"hashFunction":"SHA-256"}}}}}""" @@ -444,22 +499,25 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche val out = output.head out must beSuccessful.like { case enrichedEvent => - implicit val formats = org.json4s.DefaultFormats - val contextJ = parse(enrichedEvent.contexts) - val unstructEventJ = parse(enrichedEvent.unstruct_event) + val contextJ = parse(enrichedEvent.contexts).toOption.get.hcursor.downField("data") + val firstElem = contextJ.downArray.downField("data") + val secondElem = contextJ.downArray.right.downField("data") + val unstructEventJ = + parse(enrichedEvent.unstruct_event).toOption.get.hcursor.downField("data") + (enrichedEvent.pii must_== expected.pii) and // This is the important test, the rest just verify that nothing has changed. (enrichedEvent.app_id must_== expected.app_id) and (enrichedEvent.ip_domain must_== expected.ip_domain) and (enrichedEvent.geo_city must_== expected.geo_city) and (enrichedEvent.etl_tstamp must_== expected.etl_tstamp) and - (enrichedEvent.collector_tstamp must_== expected.collector_tstamp) and - (((contextJ \ "data")(0) \ "data" \ "emailAddress2").extract[String] must_== "bob@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress").extract[String] must_== "tim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress2").extract[String] must_== "tom@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "data" \ "emailAddress").extract[String] must_== "jim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "schema") - .extract[String] must_== "iglu:com.acme/email_sent/jsonschema/1-0-0") and - (((unstructEventJ \ "data") \ "data" \ "myVar2").extract[String] must_== "awesome") + (enrichedEvent.collector_tstamp must_== expected.collector_tstamp) + + firstElem.get[String]("emailAddress2") must_== Right("bob@acme.com") + secondElem.get[String]("emailAddress") must_== Right("tim@acme.com") + secondElem.get[String]("emailAddress2") must_== Right("tom@acme.com") + secondElem.downField("data").get[String]("emailAddress") must_== Right("jim@acme.com") + secondElem.get[String]("schema") must_== Right("iglu:com.acme/email_sent/jsonschema/1-0-0") + unstructEventJ.downField("data").get[String]("myVar2") must_== Right("awesome") } } @@ -469,39 +527,43 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche ("pii_enrichment_config" -> PiiPseudonymizerEnrichment( List( PiiJson( - fieldMutator = JsonMutators.get("contexts").get, - schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-0").toOption.get, - jsonPath = "$.['emailAddress', 'nonExistentEmailAddress']" + fieldMutator = JsonMutators.get("contexts").get, + schemaCriterion = + SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-0").toOption.get, + jsonPath = "$.['emailAddress', 'nonExistentEmailAddress']" ) ), true, - PiiStrategyPseudonymize("SHA-256", hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), "pepper123") + PiiStrategyPseudonymize( + "SHA-256", + hashFunction = DigestUtils.sha256Hex(_: Array[Byte]), + "pepper123") )) ) - val output = commonSetup(enrichmentMap = enrichmentMap) + val output = commonSetup(enrichmentMap = enrichmentMap) val expected = new EnrichedEvent() - expected.app_id = "ads" - expected.user_id = "john@acme.com" - expected.user_ipaddress = "70.46.123.145" - expected.ip_domain = null + expected.app_id = "ads" + expected.user_id = "john@acme.com" + expected.user_ipaddress = "70.46.123.145" + expected.ip_domain = null expected.user_fingerprint = "its_you_again!" - expected.geo_city = "Delray Beach" - expected.etl_tstamp = "1970-01-18 08:40:00.000" + expected.geo_city = "Delray Beach" + expected.etl_tstamp = "1970-01-18 08:40:00.000" expected.collector_tstamp = "2017-07-14 03:39:39.000" output.size must_== 1 val out = output(0) out must beSuccessful.like { case enrichedEvent => { - implicit val formats = org.json4s.DefaultFormats - val contextJ = parse(enrichedEvent.contexts) - (((contextJ \ "data")(0) \ "data" \ "emailAddress") - .extract[String] must_== "72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") and - (ScalazJson4sUtils.fieldExists(((contextJ \ "data")(0) \ "data"), "nonExistentEmailAddress") must_== false) and - (((contextJ \ "data")(0) \ "data" \ "emailAddress2") - .extract[String] must_== "bob@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress") - .extract[String] must_== "tim@acme.com") and - (((contextJ \ "data")(1) \ "data" \ "emailAddress2").extract[String] must_== "tom@acme.com") + val contextJ = parse(enrichedEvent.contexts).toOption.get.hcursor.downField("data") + val firstElem = contextJ.downArray.downField("data") + val secondElem = contextJ.downArray.right.downField("data") + + firstElem.get[String]("emailAddress") must_== + Right("72f323d5359eabefc69836369e4cabc6257c43ab6419b05dfb2211d0e44284c6") + firstElem.downField("data").get[String]("nonExistentEmailAddress") must beLeft + firstElem.get[String]("emailAddress2") must_== Right("bob@acme.com") + secondElem.get[String]("emaillAddress") must_== Right("tim@acme.com") + secondElem.get[String]("emailAddress2") must_== Right("tom@acme.com") } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala index a3c8a7cc1..bb7d64a1c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala @@ -15,8 +15,11 @@ package enrichments.registry.sqlquery import scala.collection.immutable.IntMap -import org.json4s.JObject -import org.json4s.jackson.parseJson +import cats.syntax.either._ +import io.circe._ +import io.circe.literal._ +import io.circe.parser._ +import io.circe.syntax._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers import scalaz._ @@ -43,86 +46,89 @@ class InputSpec extends Specification with ValidationMatchers { object ContextCase { val ccInput = - Input(1, pojo = None, json = JsonInput("contexts", "iglu:org.ietf/http_cookie/jsonschema/1-*-*", "$.value").some) + Input( + 1, + pojo = None, + json = JsonInput("contexts", "iglu:org.ietf/http_cookie/jsonschema/1-*-*", "$.value").some) val derInput = Input( 2, pojo = None, - json = JsonInput("derived_contexts", "iglu:org.openweathermap/weather/jsonschema/1-0-*", "$.main.humidity").some) + json = JsonInput( + "derived_contexts", + "iglu:org.openweathermap/weather/jsonschema/1-0-*", + "$.main.humidity").some) val unstructInput = - Input(3, - pojo = None, - json = JsonInput("unstruct_event", - "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", - "$.state").some) + Input( + 3, + pojo = None, + json = JsonInput( + "unstruct_event", + "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", + "$.state").some) val overrideHumidityInput = Input( 2, pojo = None, - json = JsonInput("contexts", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "$.latitude").some) + json = JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "$.latitude").some) - val derivedContext1 = parseJson( - """ - |{ - | "schema": "iglu:org.openweathermap/weather/jsonschema/1-0-0", - | "data": { - | "clouds": { - | "all": 0 - | }, - | "dt": "2014-11-10T08:38:30.000Z", - | "main": { - | "grnd_level": 1021.91, - | "humidity": 90, - | "pressure": 1021.91, - | "sea_level": 1024.77, - | "temp": 301.308, - | "temp_max": 301.308, - | "temp_min": 301.308 - | }, - | "weather": [ { "description": "Sky is Clear", "icon": "01d", "id": 800, "main": "Clear" } ], - | "wind": { - | "deg": 190.002, - | "speed": 4.39 - | } - |} - |} - """.stripMargin).asInstanceOf[JObject] + val derivedContext1 = json""" + { + "schema": "iglu:org.openweathermap/weather/jsonschema/1-0-0", + "data": { + "clouds": { + "all": 0 + }, + "dt": "2014-11-10T08:38:30.000Z", + "main": { + "grnd_level": 1021.91, + "humidity": 90, + "pressure": 1021.91, + "sea_level": 1024.77, + "temp": 301.308, + "temp_max": 301.308, + "temp_min": 301.308 + }, + "weather": [ { "description": "Sky is Clear", "icon": "01d", "id": 800, "main": "Clear" } ], + "wind": { + "deg": 190.002, + "speed": 4.39 + } + } + }""" - val cookieContext = parseJson(""" - |{ - | "schema": "iglu:org.ietf/http_cookie/jsonschema/1-0-0", - | "data": {"name": "someCookieAgain", "value": null} - |} - """.stripMargin).asInstanceOf[JObject] + val cookieContext = json""" + { + "schema": "iglu:org.ietf/http_cookie/jsonschema/1-0-0", + "data": {"name": "someCookieAgain", "value": null} + }""" - val cookieContextWithoutNull = parseJson(""" - |{ - | "schema": "iglu:org.ietf/http_cookie/jsonschema/1-0-0", - | "data": {"name": "someCookieAgain", "value": "someValue"} - |} - """.stripMargin).asInstanceOf[JObject] + val cookieContextWithoutNull = json""" + { + "schema": "iglu:org.ietf/http_cookie/jsonschema/1-0-0", + "data": {"name": "someCookieAgain", "value": "someValue"} + }""" - val unstructEvent = parseJson( - """ - |{ - | "schema": "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", - | "data": {"name": "Some EMR Job", "state": "COMPLETED"} - |} - """.stripMargin).asInstanceOf[JObject] + val unstructEvent = json""" + { + "schema": "iglu:com.snowplowanalytics.monitoring.batch/jobflow_step_status/jsonschema/1-0-0", + "data": {"name": "Some EMR Job", "state": "COMPLETED"} + }""" - val overriderContext = parseJson( - """ - |{ - | "schema": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0", - | "data": {"latitude": 43.1, "longitude": 32.1} - |} - """.stripMargin).asInstanceOf[JObject] + val overriderContext = Json.obj( + "schema" := "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0", + "data" := Json.obj( + "latitude" := Json.fromDoubleOrNull(43.1), + "longitude" := Json.fromDoubleOrNull(32.1) + ) + ) } def e1 = { val input1 = Input(1, pojo = PojoInput("user_id").some, json = None) val input2 = Input(2, pojo = PojoInput("true_tstamp").some, json = None) - val event = new EnrichedEvent + val event = new EnrichedEvent event.setUser_id("chuwy") event.setTrue_tstamp("20") @@ -138,8 +144,8 @@ class InputSpec extends Specification with ValidationMatchers { List(ccInput, derInput, unstructInput, overrideHumidityInput), event, derivedContexts = List(derivedContext1), - customContexts = List(cookieContext, overriderContext), - unstructEvent = Some(unstructEvent) + customContexts = List(cookieContext, overriderContext), + unstructEvent = Some(unstructEvent) ) placeholderMap must beSuccessful(None) } @@ -151,43 +157,42 @@ class InputSpec extends Specification with ValidationMatchers { List(ccInput, derInput, unstructInput, overrideHumidityInput), event, derivedContexts = List(derivedContext1), - customContexts = List(cookieContextWithoutNull, overriderContext), - unstructEvent = Some(unstructEvent) + customContexts = List(cookieContextWithoutNull, overriderContext), + unstructEvent = Some(unstructEvent) ) + utils.JsonPath.testMe + placeholderMap must beSuccessful( Some( - IntMap(1 -> StringPlaceholder.Value("someValue"), - 2 -> DoublePlaceholder.Value(43.1), - 3 -> StringPlaceholder.Value("COMPLETED")) + IntMap( + 1 -> StringPlaceholder.Value("someValue"), + 2 -> DoublePlaceholder.Value(43.1d), + 3 -> StringPlaceholder.Value("COMPLETED")) ) ) } def e3 = { + import ContextCase._ val jsonLatitudeInput = Input( 1, pojo = None, - json = JsonInput("contexts", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "$.latitude").some) + json = JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "$.latitude").some) val pojoLatitudeInput = Input(1, pojo = PojoInput("geo_latitude").some, json = None) - val jsonLatitudeContext = parseJson( - """ - |{ - | "schema": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-0", - | "data": {"latitude": 43.1, "longitude": 32.1} - |} - """.stripMargin).asInstanceOf[JObject] val event = new EnrichedEvent event.setGeo_latitude(42.0f) // In API Enrichment this colliding wrong - val templateContext = Input.buildPlaceholderMap(List(jsonLatitudeInput, pojoLatitudeInput), - event, - derivedContexts = Nil, - customContexts = List(jsonLatitudeContext), - unstructEvent = None) + val templateContext = Input.buildPlaceholderMap( + List(jsonLatitudeInput, pojoLatitudeInput), + event, + derivedContexts = Nil, + customContexts = List(overriderContext), + unstructEvent = None) templateContext must beSuccessful(Some(IntMap(1 -> DoublePlaceholder.Value(43.1)))) } @@ -195,30 +200,33 @@ class InputSpec extends Specification with ValidationMatchers { val invalidJsonPathInput = Input( 1, pojo = None, - json = JsonInput("contexts", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "*.invalidJsonPath").some) + json = JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "*.invalidJsonPath").some) val invalidJsonFieldInput = Input( 1, pojo = None, - json = JsonInput("invalid_field", - "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - "$.validJsonPath").some) + json = JsonInput( + "invalid_field", + "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "$.validJsonPath").some) val pojoInput = Input(1, pojo = PojoInput("app_id").some, json = None) - val templateContext = Input.buildPlaceholderMap(List(invalidJsonPathInput, pojoInput, invalidJsonFieldInput), - null, - derivedContexts = Nil, - customContexts = List(JObject(Nil)), - unstructEvent = None) + val templateContext = Input.buildPlaceholderMap( + List(invalidJsonPathInput, pojoInput, invalidJsonFieldInput), + null, + derivedContexts = Nil, + customContexts = List(Json.fromValues(Nil)), + unstructEvent = None) templateContext must beFailing.like { case errors => errors.toList must have length 3 } } def e5 = { - val event = new EnrichedEvent - val pojoInput = Input(1, pojo = PojoInput("app_id").some, json = None) + val event = new EnrichedEvent + val pojoInput = Input(1, pojo = PojoInput("app_id").some, json = None) val templateContext = pojoInput.getFromEvent(event) templateContext must beSuccessful.like { case map => map must beEqualTo((1, None)) @@ -226,14 +234,14 @@ class InputSpec extends Specification with ValidationMatchers { } def e6 = { - val event = new EnrichedEvent - val pojoInput = Input(1, pojo = PojoInput("unknown_property").some, json = None) + val event = new EnrichedEvent + val pojoInput = Input(1, pojo = PojoInput("unknown_property").some, json = None) val templateContext = pojoInput.getFromEvent(event) templateContext must beFailing } def e9 = { - val event = new EnrichedEvent + val event = new EnrichedEvent val placeholderMap = Input.buildPlaceholderMap(List(), event, Nil, Nil, None) placeholderMap must beSuccessful.like { case opt => @@ -253,10 +261,11 @@ class InputSpec extends Specification with ValidationMatchers { eventTypeMap.values.toSet.diff(typeHandlersMap.keySet) must beEmpty def e7 = { - val jsonObject = Input.extractFromJson(parseJson("""{"foo": "bar"} """)) - val jsonNull = Input.extractFromJson(parseJson("null")) - val jsonBool = Input.extractFromJson(parseJson("true")) - val jsonBigInt = Input.extractFromJson(parseJson((java.lang.Long.MAX_VALUE - 1).toString)) + val jsonObject = Input.extractFromJson(json"""{"foo": "bar"} """) + val jsonNull = Input.extractFromJson(json"null") + val jsonBool = Input.extractFromJson(json"true") + val jsonBigInt = + Input.extractFromJson(parse((java.lang.Long.MAX_VALUE - 1).toString).toOption.get) val o = jsonObject must beNone val n = jsonNull must beNone @@ -272,17 +281,20 @@ class InputSpec extends Specification with ValidationMatchers { event.setBr_viewwidth(800) event.setGeo_longitude(32.3f) - val appid = Input(3, Some(PojoInput("app_id")), None).getFromEvent(event) must beSuccessful.like { - case (3, Some(StringPlaceholder.Value("enrichment-test"))) => ok - case _ => ko - } - val viewwidth = Input(1, Some(PojoInput("br_viewwidth")), None).getFromEvent(event) must beSuccessful.like { + val appid = Input(3, Some(PojoInput("app_id")), None).getFromEvent(event) must beSuccessful + .like { + case (3, Some(StringPlaceholder.Value("enrichment-test"))) => ok + case _ => ko + } + val viewwidth = Input(1, Some(PojoInput("br_viewwidth")), None) + .getFromEvent(event) must beSuccessful.like { case (1, Some(IntPlaceholder.Value(800))) => ok - case _ => ko + case _ => ko } - val longitude = Input(1, Some(PojoInput("geo_longitude")), None).getFromEvent(event) must beSuccessful.like { + val longitude = Input(1, Some(PojoInput("geo_longitude")), None) + .getFromEvent(event) must beSuccessful.like { case (1, Some(FloatPlaceholder.Value(32.3f))) => ok - case _ => ko + case _ => ko } appid.and(viewwidth).and(longitude) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala index 4ad3da7a4..9306d95b5 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala @@ -14,8 +14,8 @@ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlque import java.sql.Date +import io.circe._ import org.joda.time.DateTime -import org.json4s._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers @@ -29,16 +29,17 @@ class OutputSpec extends Specification with ValidationMatchers { """ def e1 = - JsonOutput.getValue(1: Integer, "") must beEqualTo(JInt(1)) + JsonOutput.getValue(1: Integer, "") must beEqualTo(Json.fromInt(1)) def e2 = - JsonOutput.getValue(32.2: java.lang.Double, "") must beEqualTo(JDouble(32.2)) + JsonOutput.getValue(32.2: java.lang.Double, "") must beEqualTo(Json.fromDoubleOrNull(32.2)) def e3 = - JsonOutput.getValue(null, "") must beEqualTo(JNull) + JsonOutput.getValue(null, "") must beEqualTo(Json.Null) def e4 = { val date = new Date(1465558727000L) - JsonOutput.getValue(date, "java.sql.Date") must beEqualTo(JString(new DateTime(date).toString)) + JsonOutput.getValue(date, "java.sql.Date") must + beEqualTo(Json.fromString(new DateTime(date).toString)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala index a900e00fc..e4aa429d2 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala @@ -13,11 +13,13 @@ package com.snowplowanalytics.snowplow.enrich.common package enrichments.registry.sqlquery +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.{JsonSchemaPair, SchemaKey} -import org.json4s._ -import org.json4s.JsonDSL._ -import org.json4s.jackson.parseJson -import org.json4s.jackson.JsonMethods.asJsonNode +import io.circe._ +import io.circe.jackson.{circeToJackson, jacksonToCirce} +import io.circe.literal._ +import io.circe.parser._ +import io.circe.syntax._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers @@ -26,7 +28,7 @@ import outputs.EnrichedEvent object SqlQueryEnrichmentIntegrationTest { def continuousIntegration: Boolean = sys.env.get("CI") match { case Some("true") => true - case _ => false + case _ => false } /** @@ -34,19 +36,28 @@ object SqlQueryEnrichmentIntegrationTest { * out of *valid* JSON string and [[SchemaKey]]. * Useful only if we're passing unstruct event or custom context (but not derived) straight into * SqlQueryEnrichment.lookup method - * * WARNING: this is REQUIRED to test custom contexts (not derived!) and unstruct event */ def createPair(key: SchemaKey, validJson: String): JsonSchemaPair = { - val hierarchy = parseJson( - s"""{"rootId":null,"rootTstamp":null,"refRoot":"events","refTree":["events","${key.name}"],"refParent":"events"}""") - (key, asJsonNode(("data", parseJson(validJson)) ~ - (("hierarchy", hierarchy)) ~ - (("schema", key.toJValue)))) + val hierarchy = parse( + s"""{"rootId":null,"rootTstamp":null,"refRoot":"events","refTree":["events","${key.name}"],"refParent":"events"}""").toOption + .get + ( + key, + circeToJackson( + Json.obj( + "data" := parse(validJson).toOption.get, + "hierarchy" := hierarchy, + "schema" := jacksonToCirce(key.toJsonNode) + ) + )) } - def createDerived(key: SchemaKey, validJson: String): JObject = - (("schema", key.toSchemaUri)) ~ (("data", parseJson(validJson))) + def createDerived(key: SchemaKey, validJson: String): Json = + Json.obj( + "schema" := key.toSchemaUri, + "data" := parse(validJson).toOption.get + ) } import SqlQueryEnrichmentIntegrationTest._ @@ -60,58 +71,61 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidationMat """ val SCHEMA_KEY = - SchemaKey("com.snowplowanalytics.snowplow.enrichments", "sql_query_enrichment_config", "jsonschema", "1-0-0") + SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "sql_query_enrichment_config", + "jsonschema", + "1-0-0") def e1 = { - val configuration = parseJson(""" - |{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "sql_query_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [], - | "database": { - | "postgresql": { - | "host": "localhost", - | "port": 5432, - | "sslMode": false, - | "username": "enricher", - | "password": "supersecret1", - | "database": "sql_enrichment_test" - | } - | }, - | "query": { - | "sql": "SELECT 42 AS \"singleColumn\"" - | }, - | "output": { - | "expectedRows": "AT_MOST_ONE", - | "json": { - | "schema": "iglu:com.acme/singleColumn/jsonschema/1-0-0", - | "describes": "ALL_ROWS", - | "propertyNames": "AS_IS" - | } - | }, - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - |} - """.stripMargin) + val configuration = json""" + { + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "sql_query_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [], + "database": { + "postgresql": { + "host": "localhost", + "port": 5432, + "sslMode": false, + "username": "enricher", + "password": "supersecret1", + "database": "sql_enrichment_test" + } + }, + "query": { + "sql": "SELECT 42 AS \"singleColumn\"" + }, + "output": { + "expectedRows": "AT_MOST_ONE", + "json": { + "schema": "iglu:com.acme/singleColumn/jsonschema/1-0-0", + "describes": "ALL_ROWS", + "propertyNames": "AS_IS" + } + }, + "cache": { + "size": 3000, + "ttl": 60 + } + } + } + """ val event = new EnrichedEvent - val config = SqlQueryEnrichmentConfig.parse(configuration, SCHEMA_KEY) + val config = SqlQueryEnrichmentConfig.parse(configuration, SCHEMA_KEY) val context = config.flatMap(_.lookup(event, Nil, Nil, Nil)) - val correctContext = parseJson(""" - |{ - | "schema": "iglu:com.acme/singleColumn/jsonschema/1-0-0", - | "data": { - | "singleColumn": 42 - | } - |} - """.stripMargin) + val correctContext = json""" + { + "schema": "iglu:com.acme/singleColumn/jsonschema/1-0-0", + "data": { + "singleColumn": 42 + } + }""" context must beSuccessful.like { case List(json) => json must beEqualTo(correctContext) @@ -127,94 +141,92 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidationMat * + cache */ def e2 = { - - val configuration = parseJson( + val configuration = parse( """ - |{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "sql_query_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "placeholder": 1, - | "pojo": { - | "field": "geo_city" - | } - | }, - | - | { - | "placeholder": 2, - | "json": { - | "field": "derived_contexts", - | "schemaCriterion": "iglu:org.openweathermap/weather/jsonschema/*-*-*", - | "jsonPath": "$.dt" - | } - | }, - | - | { - | "placeholder": 3, - | "pojo": { - | "field": "user_id" - | } - | }, - | - | { - | "placeholder": 3, - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", - | "jsonPath": "$.userId" - | } - | }, - | - | { - | "placeholder": 4, - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", - | "jsonPath": "$.speed" - | } - | }, - | - | { - | "placeholder": 5, - | "json": { - | "field": "unstruct_event", - | "schemaCriterion": "iglu:com.snowplowanalytics.monitoring.kinesis/app_initialized/jsonschema/1-0-0", - | "jsonPath": "$.applicationName" - | } - | } - | ], - | - | "database": { - | "postgresql": { - | "host": "localhost", - | "port": 5432, - | "sslMode": false, - | "username": "enricher", - | "password": "supersecret1", - | "database": "sql_enrichment_test" - | } - | }, - | "query": { - | "sql": "SELECT city, country, pk FROM enrichment_test WHERE city = ? AND date_time = ? AND name = ? AND speed = ? AND aux = ?;" - | }, - | "output": { - | "expectedRows": "AT_MOST_ONE", - | "json": { - | "schema": "iglu:com.acme/demographic/jsonschema/1-0-0", - | "describes": "ALL_ROWS", - | "propertyNames": "CAMEL_CASE" - | } - | }, - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - |} - """.stripMargin) + { + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "sql_query_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "placeholder": 1, + "pojo": { + "field": "geo_city" + } + }, + + { + "placeholder": 2, + "json": { + "field": "derived_contexts", + "schemaCriterion": "iglu:org.openweathermap/weather/jsonschema/*-*-*", + "jsonPath": "$.dt" + } + }, + + { + "placeholder": 3, + "pojo": { + "field": "user_id" + } + }, + + { + "placeholder": 3, + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "jsonPath": "$.userId" + } + }, + + { + "placeholder": 4, + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-1-*", + "jsonPath": "$.speed" + } + }, + + { + "placeholder": 5, + "json": { + "field": "unstruct_event", + "schemaCriterion": "iglu:com.snowplowanalytics.monitoring.kinesis/app_initialized/jsonschema/1-0-0", + "jsonPath": "$.applicationName" + } + } + ], + + "database": { + "postgresql": { + "host": "localhost", + "port": 5432, + "sslMode": false, + "username": "enricher", + "password": "supersecret1", + "database": "sql_enrichment_test" + } + }, + "query": { + "sql": "SELECT city, country, pk FROM enrichment_test WHERE city = ? AND date_time = ? AND name = ? AND speed = ? AND aux = ?;" + }, + "output": { + "expectedRows": "AT_MOST_ONE", + "json": { + "schema": "iglu:com.acme/demographic/jsonschema/1-0-0", + "describes": "ALL_ROWS", + "propertyNames": "CAMEL_CASE" + } + }, + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get val event1 = new EnrichedEvent event1.setGeo_city("Krasnoyarsk") @@ -227,7 +239,11 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidationMat SchemaKey("com.snowplowanalytics.snowplow", "geolocation_context", "jsonschema", "1-1-0"), """ {"latitude": 12.5, "longitude": 32.1, "speed": 10.0} """) val ue1 = createPair( - SchemaKey("com.snowplowanalytics.monitoring.kinesis", "app_initialized", "jsonschema", "1-0-0"), + SchemaKey( + "com.snowplowanalytics.monitoring.kinesis", + "app_initialized", + "jsonschema", + "1-0-0"), """ {"applicationName": "ue_test_krsk"} """) val event2 = new EnrichedEvent @@ -241,7 +257,11 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidationMat SchemaKey("com.snowplowanalytics.snowplow", "geolocation_context", "jsonschema", "1-1-0"), """ {"latitude": 12.5, "longitude": 32.1, "speed": 25.0} """) val ue2 = createPair( - SchemaKey("com.snowplowanalytics.monitoring.kinesis", "app_initialized", "jsonschema", "1-0-0"), + SchemaKey( + "com.snowplowanalytics.monitoring.kinesis", + "app_initialized", + "jsonschema", + "1-0-0"), """ {"applicationName": "ue_test_london"} """) val event3 = new EnrichedEvent @@ -255,7 +275,11 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidationMat SchemaKey("com.snowplowanalytics.snowplow", "geolocation_context", "jsonschema", "1-1-0"), """ {"latitude": 12.5, "longitude": 32.1, "speed": 2.5} """) val ue3 = createPair( - SchemaKey("com.snowplowanalytics.monitoring.kinesis", "app_initialized", "jsonschema", "1-0-0"), + SchemaKey( + "com.snowplowanalytics.monitoring.kinesis", + "app_initialized", + "jsonschema", + "1-0-0"), """ {"applicationName": "ue_test_ny"} """) val event4 = new EnrichedEvent @@ -273,45 +297,77 @@ class SqlQueryEnrichmentIntegrationTest extends Specification with ValidationMat SchemaKey("com.snowplowanalytics.snowplow", "geolocation_context", "jsonschema", "1-1-0"), """ {"latitude": 12.5, "longitude": 32.1, "speed": 25.0} """) val ue4 = createPair( - SchemaKey("com.snowplowanalytics.monitoring.kinesis", "app_initialized", "jsonschema", "1-0-0"), + SchemaKey( + "com.snowplowanalytics.monitoring.kinesis", + "app_initialized", + "jsonschema", + "1-0-0"), """ {"applicationName": "ue_test_london"} """) val config = SqlQueryEnrichmentConfig.parse(configuration, SCHEMA_KEY) - val context1 = config.flatMap(_.lookup(event1, List(weatherContext1), List(geoContext1), List(ue1))) - val result_context1 = parseJson("""|{"schema":"iglu:com.acme/demographic/jsonschema/1-0-0", - | "data": { - | "city": "Krasnoyarsk", - | "country": "Russia", - | "pk": 1}}""".stripMargin) + val context1 = + config.flatMap(_.lookup(event1, List(weatherContext1), List(geoContext1), List(ue1))) + val result_context1 = json""" + { + "schema":"iglu:com.acme/demographic/jsonschema/1-0-0", + "data": { + "city": "Krasnoyarsk", + "country": "Russia", + "pk": 1 + } + }""" - val context2 = config.flatMap(_.lookup(event2, List(weatherContext2), List(geoContext2), List(ue2))) - val result_context2 = parseJson("""|{"schema":"iglu:com.acme/demographic/jsonschema/1-0-0", - | "data": { - | "city": "London", - | "country": "England", - | "pk": 2}}""".stripMargin) + val context2 = + config.flatMap(_.lookup(event2, List(weatherContext2), List(geoContext2), List(ue2))) + val result_context2 = json""" + { + "schema":"iglu:com.acme/demographic/jsonschema/1-0-0", + "data": { + "city": "London", + "country": "England", + "pk": 2 + } + }""" - val context3 = config.flatMap(_.lookup(event3, List(weatherContext3), List(geoContext3), List(ue3))) - val result_context3 = parseJson("""|{"schema":"iglu:com.acme/demographic/jsonschema/1-0-0", - | "data": { - | "city": "New York", - | "country": "USA", - | "pk": 3}} - """.stripMargin) + val context3 = + config.flatMap(_.lookup(event3, List(weatherContext3), List(geoContext3), List(ue3))) + val result_context3 = json""" + { + "schema":"iglu:com.acme/demographic/jsonschema/1-0-0", + "data": { + "city": "New York", + "country": "USA", + "pk": 3 + } + }""" - val context4 = config.flatMap(_.lookup(event4, List(weatherContext4), List(geoContext4, clientSession4), List(ue4))) - val result_context4 = parseJson("""|{"schema":"iglu:com.acme/demographic/jsonschema/1-0-0", - | "data": { - | "city": "London", - | "country": "England", - | "pk": 2}}""".stripMargin) + val context4 = config.flatMap( + _.lookup(event4, List(weatherContext4), List(geoContext4, clientSession4), List(ue4))) + val result_context4 = json""" + { + "schema":"iglu:com.acme/demographic/jsonschema/1-0-0", + "data": { + "city": "London", + "country": "England", + "pk": 2 + } + }""" - val res1 = context1 must beSuccessful.like { case List(ctx) => ctx must beEqualTo(result_context1) } - val res2 = context2 must beSuccessful.like { case List(ctx) => ctx must beEqualTo(result_context2) } - val res3 = context3 must beSuccessful.like { case List(ctx) => ctx must beEqualTo(result_context3) } - val res4 = context4 must beSuccessful.like { case List(ctx) => ctx must beEqualTo(result_context4) } - val cache = config.map(_.cache.actualLoad) must beSuccessful.like { case size => size must beEqualTo(3) } + val res1 = context1 must beSuccessful.like { + case List(ctx) => ctx must beEqualTo(result_context1) + } + val res2 = context2 must beSuccessful.like { + case List(ctx) => ctx must beEqualTo(result_context2) + } + val res3 = context3 must beSuccessful.like { + case List(ctx) => ctx must beEqualTo(result_context3) + } + val res4 = context4 must beSuccessful.like { + case List(ctx) => ctx must beEqualTo(result_context4) + } + val cache = config.map(_.cache.actualLoad) must + beSuccessful.like { case size => size must beEqualTo(3) } res1.and(res2).and(res3).and(res4).and(cache) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala index e4998dac9..28b7c4366 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala @@ -12,8 +12,9 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.sqlquery +import cats.syntax.either._ import com.snowplowanalytics.iglu.client.SchemaKey -import org.json4s.jackson.parseJson +import io.circe.parser._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers @@ -26,7 +27,11 @@ class SqlQueryEnrichmentSpec extends Specification with ValidationMatchers { """ val SCHEMA_KEY = - SchemaKey("com.snowplowanalytics.snowplow.enrichments", "sql_query_enrichment_config", "jsonschema", "1-0-0") + SchemaKey( + "com.snowplowanalytics.snowplow.enrichments", + "sql_query_enrichment_config", + "jsonschema", + "1-0-0") def e1 = { val inputs = List( @@ -35,195 +40,207 @@ class SqlQueryEnrichmentSpec extends Specification with ValidationMatchers { 1, pojo = None, json = Some( - JsonInput("contexts", "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", "$.userId"))), + JsonInput( + "contexts", + "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "$.userId"))), Input(2, pojo = Some(PojoInput("app_id")), json = None) ) val db = Db( postgresql = Some( - PostgresqlDb("cluster01.redshift.acme.com", 5439, sslMode = true, "snowplow_enrich_ro", "1asIkJed", "crm")), + PostgresqlDb( + "cluster01.redshift.acme.com", + 5439, + sslMode = true, + "snowplow_enrich_ro", + "1asIkJed", + "crm")), mysql = None) val output = JsonOutput("iglu:com.acme/user/jsonschema/1-0-0", "ALL_ROWS", "CAMEL_CASE") - val cache = Cache(3000, 60) + val cache = Cache(3000, 60) val query = Query( "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1") val config = SqlQueryEnrichment(inputs, db, query, Output(output, "AT_MOST_ONE"), cache) - val configuration = parseJson( - """|{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "sql_query_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "placeholder": 1, - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "placeholder": 1, - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", - | "jsonPath": "$.userId" - | } - | }, - | { - | "placeholder": 2, - | "pojo": { - | "field": "app_id" - | } - | } - | ], - | "query": { - | "sql": "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1" - | }, - | "database": { - | "postgresql": { - | "host": "cluster01.redshift.acme.com", - | "port": 5439, - | "sslMode": true, - | "username": "snowplow_enrich_ro", - | "password": "1asIkJed", - | "database": "crm" - | } - | }, - | "output": { - | "expectedRows": "AT_MOST_ONE", - | "json": { - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "describes": "ALL_ROWS", - | "propertyNames": "CAMEL_CASE" - | } - | }, - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | }""".stripMargin) + val configuration = parse( + """ + { + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "sql_query_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "placeholder": 1, + "pojo": { + "field": "user_id" + } + }, + { + "placeholder": 1, + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "jsonPath": "$.userId" + } + }, + { + "placeholder": 2, + "pojo": { + "field": "app_id" + } + } + ], + "query": { + "sql": "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1" + }, + "database": { + "postgresql": { + "host": "cluster01.redshift.acme.com", + "port": 5439, + "sslMode": true, + "username": "snowplow_enrich_ro", + "password": "1asIkJed", + "database": "crm" + } + }, + "output": { + "expectedRows": "AT_MOST_ONE", + "json": { + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "describes": "ALL_ROWS", + "propertyNames": "CAMEL_CASE" + } + }, + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get SqlQueryEnrichmentConfig.parse(configuration, SCHEMA_KEY) must beSuccessful(config) } def e2 = { // $.output.json.describes contains invalid value - val configuration = parseJson( - """|{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "sql_query_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "placeholder": 1, - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "placeholder": 1, - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", - | "jsonPath": "$.userId" - | } - | }, - | { - | "placeholder": 2, - | "pojo": { - | "field": "app_id" - | } - | } - | ], - | "query": { - | "sql": "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1" - | }, - | "database": { - | "postgresql": { - | "host": "cluster01.redshift.acme.com", - | "port": 5439, - | "sslMode": true, - | "username": "snowplow_enrich_ro", - | "password": "1asIkJed", - | "database": "crm" - | } - | }, - | "output": { - | "expectedRows": "AT_MOST_ONE", - | "json": { - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "describes": "INVALID", - | "propertyNames": "CAMEL_CASE" - | } - | }, - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | }""".stripMargin) + val configuration = parse( + """ + { + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "sql_query_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "placeholder": 1, + "pojo": { + "field": "user_id" + } + }, + { + "placeholder": 1, + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "jsonPath": "$.userId" + } + }, + { + "placeholder": 2, + "pojo": { + "field": "app_id" + } + } + ], + "query": { + "sql": "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1" + }, + "database": { + "postgresql": { + "host": "cluster01.redshift.acme.com", + "port": 5439, + "sslMode": true, + "username": "snowplow_enrich_ro", + "password": "1asIkJed", + "database": "crm" + } + }, + "output": { + "expectedRows": "AT_MOST_ONE", + "json": { + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "describes": "INVALID", + "propertyNames": "CAMEL_CASE" + } + }, + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get SqlQueryEnrichmentConfig.parse(configuration, SCHEMA_KEY) must beFailing } def e3 = { - val configuration = parseJson( - """|{ - | "vendor": "com.snowplowanalytics.snowplow.enrichments", - | "name": "sql_query_enrichment_config", - | "enabled": true, - | "parameters": { - | "inputs": [ - | { - | "placeholder": 1, - | "pojo": { - | "field": "user_id" - | } - | }, - | { - | "placeholder": 1, - | "json": { - | "field": "contexts", - | "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", - | "jsonPath": "$.userId" - | } - | }, - | { - | "placeholder": 2, - | "pojo": { - | "field": "app_id" - | } - | } - | ], - | "query": { - | "sql": "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1" - | }, - | "database": { - | "mysql": { - | "host": "cluster01.redshift.acme.com", - | "port": 5439, - | "sslMode": true, - | "username": "snowplow_enrich_ro", - | "password": "1asIkJed", - | "database": "crm" - | } - | }, - | "output": { - | "expectedRows": "AT_LEAST_ONE", - | "json": { - | "schema": "iglu:com.acme/user/jsonschema/1-0-0", - | "describes": "EVERY_ROW", - | "propertyNames": "CAMEL_CASE" - | } - | }, - | "cache": { - | "size": 3000, - | "ttl": 60 - | } - | } - | }""".stripMargin) + val configuration = parse( + """ + { + "vendor": "com.snowplowanalytics.snowplow.enrichments", + "name": "sql_query_enrichment_config", + "enabled": true, + "parameters": { + "inputs": [ + { + "placeholder": 1, + "pojo": { + "field": "user_id" + } + }, + { + "placeholder": 1, + "json": { + "field": "contexts", + "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*", + "jsonPath": "$.userId" + } + }, + { + "placeholder": 2, + "pojo": { + "field": "app_id" + } + } + ], + "query": { + "sql": "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1" + }, + "database": { + "mysql": { + "host": "cluster01.redshift.acme.com", + "port": 5439, + "sslMode": true, + "username": "snowplow_enrich_ro", + "password": "1asIkJed", + "database": "crm" + } + }, + "output": { + "expectedRows": "AT_LEAST_ONE", + "json": { + "schema": "iglu:com.acme/user/jsonschema/1-0-0", + "describes": "EVERY_ROW", + "propertyNames": "CAMEL_CASE" + } + }, + "cache": { + "size": 3000, + "ttl": 60 + } + } + }""").toOption.get SqlQueryEnrichmentConfig.parse(configuration, SCHEMA_KEY) must beSuccessful } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala index 5790faa1e..7944f69cb 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala @@ -34,16 +34,16 @@ class ExtractPageUriSpec extends Specification with DataTables with ValidationMa // Valid URI combinations val originalUri = "http://www.mysite.com/shop/session/_internal/checkout" - val customUri = "http://www.mysite.com/shop/checkout" // E.g. set by setCustomUrl in JS Tracker + val customUri = "http://www.mysite.com/shop/checkout" // E.g. set by setCustomUrl in JS Tracker val originalURI = new URI(originalUri) - val customURI = new URI(customUri) + val customURI = new URI(customUri) def e2 = - "SPEC NAME" || "URI TAKEN FROM COLLECTOR'S REFERER" | "URI SENT BY TRACKER" | "EXPECTED URI" | - "both URIs match (98% of the time)" !! originalUri.some ! originalUri.some ! originalURI.some | - "tracker didn't send URI (e.g. No-JS Tracker)" !! originalUri.some ! None ! originalURI.some | - "collector didn't record the referer (rare)" !! None ! originalUri.some ! originalURI.some | - "collector and tracker URIs differ - use tracker" !! originalUri.some ! customUri.some ! customURI.some |> { + "SPEC NAME" || "URI TAKEN FROM COLLECTOR'S REFERER" | "URI SENT BY TRACKER" | "EXPECTED URI" | + "both URIs match (98% of the time)" !! originalUri.some ! originalUri.some ! originalURI.some | + "tracker didn't send URI (e.g. No-JS Tracker)" !! originalUri.some ! None ! originalURI.some | + "collector didn't record the referer (rare)" !! None ! originalUri.some ! originalURI.some | + "collector and tracker URIs differ - use tracker" !! originalUri.some ! customUri.some ! customURI.some |> { (_, fromReferer, fromTracker, expected) => PageEnrichments.extractPageUri(fromReferer, fromTracker) must beSuccessful(expected) @@ -55,5 +55,6 @@ class ExtractPageUriSpec extends Specification with DataTables with ValidationMa // See https://github.com/snowplow/snowplow/issues/268 for background behind this test def e3 = - PageEnrichments.extractPageUri(originalUri.some, truncatedUri.some) must beSuccessful(truncatedURI.some) + PageEnrichments.extractPageUri(originalUri.some, truncatedUri.some) must beSuccessful( + truncatedURI.some) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala index 77f77f9d3..4f620a19c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala @@ -31,7 +31,8 @@ class ParseCrossDomainSpec extends Specification with DataTables with Validation PageEnrichments.parseCrossDomain(Map()) must beSuccessful((None, None)) def e2 = { - val expected = "Field [sp_dtm]: [not-a-timestamp] is not in the expected format (ms since epoch)" + val expected = + "Field [sp_dtm]: [not-a-timestamp] is not in the expected format (ms since epoch)" PageEnrichments.parseCrossDomain(Map("_sp" -> "abc.not-a-timestamp")) must beFailing(expected) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CljTomcatLoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CljTomcatLoaderSpec.scala index 398a9a172..fcd620b3c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CljTomcatLoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CljTomcatLoaderSpec.scala @@ -22,7 +22,11 @@ import Scalaz._ import SpecHelpers._ -class CljTomcatLoaderSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class CljTomcatLoaderSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the CljTomcatLoader functionality toCollectorPayload should return a CanonicalInput for a valid raw event $e1 @@ -33,150 +37,152 @@ class CljTomcatLoaderSpec extends Specification with DataTables with ValidationM object Expected { val collector = "clj-tomcat" - val encoding = "UTF-8" - val vendor = "com.snowplowanalytics.snowplow" + val encoding = "UTF-8" + val vendor = "com.snowplowanalytics.snowplow" val ipAddress = "37.157.33.123".some } def e1 = - "SPEC NAME" || "RAW" | "EXP. VERSION" | "EXP. PAYLOAD" | "EXP. CONTENT TYPE" | "EXP. BODY" | "EXP. TIMESTAMP" | "EXP. USER AGENT" | "EXP. REFERER URI" | + "SPEC NAME" || "RAW" | "EXP. VERSION" | "EXP. PAYLOAD" | "EXP. CONTENT TYPE" | "EXP. BODY" | "EXP. TIMESTAMP" | "EXP. USER AGENT" | "EXP. REFERER URI" | "Snowplow Tp1 GET w/ v0.6.0 collector" !! "2013-08-29 00:18:48 - 830 37.157.33.123 GET d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/analytics/index.html Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0 e=pv&page=Introduction%20-%20Snowplow%20Analytics%25&dtm=1377735557970&tid=567074&vp=1024x635&ds=1024x635&vid=1&duid=7969620089de36eb&p=web&tv=js-0.12.0&fp=308909339&aid=snowplowweb&lang=en-US&cs=UTF-8&tz=America%2FLos_Angeles&refr=http%3A%2F%2Fwww.metacrawler.com%2Fsearch%2Fweb%3Ffcoid%3D417%26fcop%3Dtopnav%26fpid%3D27%26q%3Dsnowplow%2Banalytics%26ql%3D&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1024x768&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fanalytics%2Findex.html - - -" ! "tp1" ! toNameValuePairs( - "e" -> "pv", - "page" -> "Introduction - Snowplow Analytics%", - "dtm" -> "1377735557970", - "tid" -> "567074", - "vp" -> "1024x635", - "ds" -> "1024x635", - "vid" -> "1", - "duid" -> "7969620089de36eb", - "p" -> "web", - "tv" -> "js-0.12.0", - "fp" -> "308909339", - "aid" -> "snowplowweb", - "lang" -> "en-US", - "cs" -> "UTF-8", - "tz" -> "America/Los_Angeles", - "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", - "f_pdf" -> "1", - "f_qt" -> "1", + "e" -> "pv", + "page" -> "Introduction - Snowplow Analytics%", + "dtm" -> "1377735557970", + "tid" -> "567074", + "vp" -> "1024x635", + "ds" -> "1024x635", + "vid" -> "1", + "duid" -> "7969620089de36eb", + "p" -> "web", + "tv" -> "js-0.12.0", + "fp" -> "308909339", + "aid" -> "snowplowweb", + "lang" -> "en-US", + "cs" -> "UTF-8", + "tz" -> "America/Los_Angeles", + "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", + "f_pdf" -> "1", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "1", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "1", + "f_wma" -> "1", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "1", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "1024x768", - "cd" -> "24", - "cookie" -> "1", - "url" -> "http://snowplowanalytics.com/analytics/index.html" - ) ! None ! None ! DateTime.parse("2013-08-29T00:18:48.000+00:00") ! + "f_ag" -> "0", + "res" -> "1024x768", + "cd" -> "24", + "cookie" -> "1", + "url" -> "http://snowplowanalytics.com/analytics/index.html" + ) ! None ! None ! DateTime.parse("2013-08-29T00:18:48.000+00:00") ! "Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0".some ! "http://snowplowanalytics.com/analytics/index.html".some | - "Snowplow Tp1 GET w/ v0.7.0 collector" !! "2013-08-29 00:18:48 - 830 37.157.33.123 GET d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/analytics/index.html Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0 e=pv&page=Introduction%20-%20Snowplow%20Analytics%25&dtm=1377735557970&tid=567074&vp=1024x635&ds=1024x635&vid=1&duid=7969620089de36eb&p=web&tv=js-0.12.0&fp=308909339&aid=snowplowweb&lang=en-US&cs=UTF-8&tz=America%2FLos_Angeles&refr=http%3A%2F%2Fwww.metacrawler.com%2Fsearch%2Fweb%3Ffcoid%3D417%26fcop%3Dtopnav%26fpid%3D27%26q%3Dsnowplow%2Banalytics%26ql%3D&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1024x768&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fanalytics%2Findex.html - - - - -" ! + "Snowplow Tp1 GET w/ v0.7.0 collector" !! "2013-08-29 00:18:48 - 830 37.157.33.123 GET d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/analytics/index.html Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0 e=pv&page=Introduction%20-%20Snowplow%20Analytics%25&dtm=1377735557970&tid=567074&vp=1024x635&ds=1024x635&vid=1&duid=7969620089de36eb&p=web&tv=js-0.12.0&fp=308909339&aid=snowplowweb&lang=en-US&cs=UTF-8&tz=America%2FLos_Angeles&refr=http%3A%2F%2Fwww.metacrawler.com%2Fsearch%2Fweb%3Ffcoid%3D417%26fcop%3Dtopnav%26fpid%3D27%26q%3Dsnowplow%2Banalytics%26ql%3D&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1024x768&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fanalytics%2Findex.html - - - - -" ! "tp1" ! toNameValuePairs( - "e" -> "pv", - "page" -> "Introduction - Snowplow Analytics%", - "dtm" -> "1377735557970", - "tid" -> "567074", - "vp" -> "1024x635", - "ds" -> "1024x635", - "vid" -> "1", - "duid" -> "7969620089de36eb", - "p" -> "web", - "tv" -> "js-0.12.0", - "fp" -> "308909339", - "aid" -> "snowplowweb", - "lang" -> "en-US", - "cs" -> "UTF-8", - "tz" -> "America/Los_Angeles", - "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", - "f_pdf" -> "1", - "f_qt" -> "1", + "e" -> "pv", + "page" -> "Introduction - Snowplow Analytics%", + "dtm" -> "1377735557970", + "tid" -> "567074", + "vp" -> "1024x635", + "ds" -> "1024x635", + "vid" -> "1", + "duid" -> "7969620089de36eb", + "p" -> "web", + "tv" -> "js-0.12.0", + "fp" -> "308909339", + "aid" -> "snowplowweb", + "lang" -> "en-US", + "cs" -> "UTF-8", + "tz" -> "America/Los_Angeles", + "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", + "f_pdf" -> "1", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "1", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "1", + "f_wma" -> "1", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "1", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "1024x768", - "cd" -> "24", - "cookie" -> "1", - "url" -> "http://snowplowanalytics.com/analytics/index.html" - ) ! None ! None ! DateTime.parse("2013-08-29T00:18:48.000+00:00") ! + "f_ag" -> "0", + "res" -> "1024x768", + "cd" -> "24", + "cookie" -> "1", + "url" -> "http://snowplowanalytics.com/analytics/index.html" + ) ! None ! None ! DateTime.parse("2013-08-29T00:18:48.000+00:00") ! "Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0".some ! "http://snowplowanalytics.com/analytics/index.html".some | - "Snowplow Tp2 POST w/ v0.6.0 collector" !! "2014-09-08 13:59:07 - - 37.157.33.123 POST - /com.snowplowanalytics.snowplow/tp2 200 - python-requests%2F2.2.1+CPython%2F3.3.5+Linux%2F3.2.0-61-generic &cv=clj-0.7.0-tom-0.1.0&nuid=5c6c40e4-eff8-409b-9327-471f303e30b6 - - - application%2Fjson%3B+charset%3Dutf-8 eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvcGF5bG9hZF9kYXRhL2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IFt7ImR0bSI6ICIxNDEwMTg0NzQ2ODk0IiwgImUiOiAicHYiLCAiZWlkIjogIjJjYWU0MTkxLTMxY2QtNDc4My04MmE4LWRmNTMxOGY0NGFmZiIsICJ1cmwiOiAiaHR0cDovL3d3dy5leGFtcGxlLmNvbSIsICJ0diI6ICJweS0wLjUuMCIsICJjeCI6ICJleUp6WTJobGJXRWlPaUFpYVdkc2RUcGpiMjB1YzI1dmQzQnNiM2RoYm1Gc2VYUnBZM011YzI1dmQzQnNiM2N2WTI5dWRHVjRkSE12YW5OdmJuTmphR1Z0WVM4eExUQXRNQ0lzSUNKa1lYUmhJam9nVzNzaWMyTm9aVzFoSWpvZ0ltbG5iSFU2WTI5dExuTnViM2R3Ykc5M1lXNWhiSGwwYVdOekxuTnViM2R3Ykc5M0wyMXZZbWxzWlY5amIyNTBaWGgwTDJwemIyNXpZMmhsYldFdk1TMHdMVEFpTENBaVpHRjBZU0k2SUhzaVpHVjJhV05sVFdGdWRXWmhZM1IxY21WeUlqb2dJa0Z0YzNSeVlXUWlMQ0FpWVc1a2NtOXBaRWxrWm1FaU9pQWljMjl0WlY5aGJtUnliMmxrU1dSbVlTSXNJQ0prWlhacFkyVk5iMlJsYkNJNklDSnNZWEpuWlNJc0lDSnZjR1Z1U1dSbVlTSTZJQ0p6YjIxbFgwbGtabUVpTENBaVkyRnljbWxsY2lJNklDSnpiMjFsWDJOaGNuSnBaWElpTENBaVlYQndiR1ZKWkdaaElqb2dJbk52YldWZllYQndiR1ZKWkdaaElpd2dJbTl6Vm1WeWMybHZiaUk2SUNJekxqQXVNQ0lzSUNKaGNIQnNaVWxrWm5ZaU9pQWljMjl0WlY5aGNIQnNaVWxrWm5ZaUxDQWliM05VZVhCbElqb2dJazlUV0NKOWZTd2dleUp6WTJobGJXRWlPaUFpYVdkc2RUcGpiMjB1YzI1dmQzQnNiM2RoYm1Gc2VYUnBZM011YzI1dmQzQnNiM2N2WjJWdmJHOWpZWFJwYjI1ZlkyOXVkR1Y0ZEM5cWMyOXVjMk5vWlcxaEx6RXRNQzB3SWl3Z0ltUmhkR0VpT2lCN0lteHZibWRwZEhWa1pTSTZJREV3TENBaVlXeDBhWFIxWkdWQlkyTjFjbUZqZVNJNklEQXVNeXdnSW14aGRHbDBkV1JsSWpvZ055d2dJbXhoZEdsMGRXUmxURzl1WjJsMGRXUmxRV05qZFhKaFkza2lPaUF3TGpVc0lDSmlaV0Z5YVc1bklqb2dOVEFzSUNKaGJIUnBkSFZrWlNJNklESXdMQ0FpYzNCbFpXUWlPaUF4Tm4xOVhYMD0iLCAicCI6ICJwYyJ9LCB7ImR0bSI6ICIxNDEwMTg0NzQ2ODk0IiwgImUiOiAic2UiLCAiZWlkIjogIjVhNzExODg1LTY5ZGMtNGY0Mi04Nzg1LWZjNjVmMTc1OGVjMCIsICJzZV9hYyI6ICJteV9hY3Rpb24iLCAidHYiOiAicHktMC41LjAiLCAiY3giOiAiZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdlkyOXVkR1Y0ZEhNdmFuTnZibk5qYUdWdFlTOHhMVEF0TUNJc0lDSmtZWFJoSWpvZ1czc2ljMk5vWlcxaElqb2dJbWxuYkhVNlkyOXRMbk51YjNkd2JHOTNZVzVoYkhsMGFXTnpMbk51YjNkd2JHOTNMMjF2WW1sc1pWOWpiMjUwWlhoMEwycHpiMjV6WTJobGJXRXZNUzB3TFRBaUxDQWlaR0YwWVNJNklIc2laR1YyYVdObFRXRnVkV1poWTNSMWNtVnlJam9nSWtGdGMzUnlZV1FpTENBaVlXNWtjbTlwWkVsa1ptRWlPaUFpYzI5dFpWOWhibVJ5YjJsa1NXUm1ZU0lzSUNKa1pYWnBZMlZOYjJSbGJDSTZJQ0pzWVhKblpTSXNJQ0p2Y0dWdVNXUm1ZU0k2SUNKemIyMWxYMGxrWm1FaUxDQWlZMkZ5Y21sbGNpSTZJQ0p6YjIxbFgyTmhjbkpwWlhJaUxDQWlZWEJ3YkdWSlpHWmhJam9nSW5OdmJXVmZZWEJ3YkdWSlpHWmhJaXdnSW05elZtVnljMmx2YmlJNklDSXpMakF1TUNJc0lDSmhjSEJzWlVsa1puWWlPaUFpYzI5dFpWOWhjSEJzWlVsa1puWWlMQ0FpYjNOVWVYQmxJam9nSWs5VFdDSjlmU3dnZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdloyVnZiRzlqWVhScGIyNWZZMjl1ZEdWNGRDOXFjMjl1YzJOb1pXMWhMekV0TUMwd0lpd2dJbVJoZEdFaU9pQjdJbXh2Ym1kcGRIVmtaU0k2SURFd0xDQWlZV3gwYVhSMVpHVkJZMk4xY21GamVTSTZJREF1TXl3Z0lteGhkR2wwZFdSbElqb2dOeXdnSW14aGRHbDBkV1JsVEc5dVoybDBkV1JsUVdOamRYSmhZM2tpT2lBd0xqVXNJQ0ppWldGeWFXNW5Jam9nTlRBc0lDSmhiSFJwZEhWa1pTSTZJREl3TENBaWMzQmxaV1FpT2lBeE5uMTlYWDA9IiwgInNlX2NhIjogIm15X2NhdGVnb3J5IiwgInAiOiAicGMifSwgeyJkdG0iOiAiMTQxMDE4NDc0Njg5NSIsICJlIjogInNlIiwgImVpZCI6ICI4M2VhYzIyNy03MTI5LTQyYTctYWY0NS00MGY2M2VkNGI5ZGQiLCAic2VfYWMiOiAiYW5vdGhlcl9hY3Rpb24iLCAidHYiOiAicHktMC41LjAiLCAiY3giOiAiZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdlkyOXVkR1Y0ZEhNdmFuTnZibk5qYUdWdFlTOHhMVEF0TUNJc0lDSmtZWFJoSWpvZ1czc2ljMk5vWlcxaElqb2dJbWxuYkhVNlkyOXRMbk51YjNkd2JHOTNZVzVoYkhsMGFXTnpMbk51YjNkd2JHOTNMMjF2WW1sc1pWOWpiMjUwWlhoMEwycHpiMjV6WTJobGJXRXZNUzB3TFRBaUxDQWlaR0YwWVNJNklIc2laR1YyYVdObFRXRnVkV1poWTNSMWNtVnlJam9nSWtGdGMzUnlZV1FpTENBaVlXNWtjbTlwWkVsa1ptRWlPaUFpYzI5dFpWOWhibVJ5YjJsa1NXUm1ZU0lzSUNKa1pYWnBZMlZOYjJSbGJDSTZJQ0pzWVhKblpTSXNJQ0p2Y0dWdVNXUm1ZU0k2SUNKemIyMWxYMGxrWm1FaUxDQWlZMkZ5Y21sbGNpSTZJQ0p6YjIxbFgyTmhjbkpwWlhJaUxDQWlZWEJ3YkdWSlpHWmhJam9nSW5OdmJXVmZZWEJ3YkdWSlpHWmhJaXdnSW05elZtVnljMmx2YmlJNklDSXpMakF1TUNJc0lDSmhjSEJzWlVsa1puWWlPaUFpYzI5dFpWOWhjSEJzWlVsa1puWWlMQ0FpYjNOVWVYQmxJam9nSWs5VFdDSjlmU3dnZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdloyVnZiRzlqWVhScGIyNWZZMjl1ZEdWNGRDOXFjMjl1YzJOb1pXMWhMekV0TUMwd0lpd2dJbVJoZEdFaU9pQjdJbXh2Ym1kcGRIVmtaU0k2SURFd0xDQWlZV3gwYVhSMVpHVkJZMk4xY21GamVTSTZJREF1TXl3Z0lteGhkR2wwZFdSbElqb2dOeXdnSW14aGRHbDBkV1JsVEc5dVoybDBkV1JsUVdOamRYSmhZM2tpT2lBd0xqVXNJQ0ppWldGeWFXNW5Jam9nTlRBc0lDSmhiSFJwZEhWa1pTSTZJREl3TENBaWMzQmxaV1FpT2lBeE5uMTlYWDA9IiwgInNlX2NhIjogImFub3RoZXJfY2F0ZWdvcnkiLCAicCI6ICJwYyJ9XX0" ! - "tp2" ! toNameValuePairs("cv" -> "clj-0.7.0-tom-0.1.0", "nuid" -> "5c6c40e4-eff8-409b-9327-471f303e30b6") ! "application/json; charset=utf-8".some ! """{"schema": "iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-0", "data": [{"dtm": "1410184746894", "e": "pv", "eid": "2cae4191-31cd-4783-82a8-df5318f44aff", "url": "http://www.example.com", "tv": "py-0.5.0", "cx": "eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY29udGV4dHMvanNvbnNjaGVtYS8xLTAtMCIsICJkYXRhIjogW3sic2NoZW1hIjogImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IHsiZGV2aWNlTWFudWZhY3R1cmVyIjogIkFtc3RyYWQiLCAiYW5kcm9pZElkZmEiOiAic29tZV9hbmRyb2lkSWRmYSIsICJkZXZpY2VNb2RlbCI6ICJsYXJnZSIsICJvcGVuSWRmYSI6ICJzb21lX0lkZmEiLCAiY2FycmllciI6ICJzb21lX2NhcnJpZXIiLCAiYXBwbGVJZGZhIjogInNvbWVfYXBwbGVJZGZhIiwgIm9zVmVyc2lvbiI6ICIzLjAuMCIsICJhcHBsZUlkZnYiOiAic29tZV9hcHBsZUlkZnYiLCAib3NUeXBlIjogIk9TWCJ9fSwgeyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwgImRhdGEiOiB7ImxvbmdpdHVkZSI6IDEwLCAiYWx0aXR1ZGVBY2N1cmFjeSI6IDAuMywgImxhdGl0dWRlIjogNywgImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOiAwLjUsICJiZWFyaW5nIjogNTAsICJhbHRpdHVkZSI6IDIwLCAic3BlZWQiOiAxNn19XX0=", "p": "pc"}, {"dtm": "1410184746894", "e": "se", "eid": "5a711885-69dc-4f42-8785-fc65f1758ec0", "se_ac": "my_action", "tv": "py-0.5.0", "cx": "eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY29udGV4dHMvanNvbnNjaGVtYS8xLTAtMCIsICJkYXRhIjogW3sic2NoZW1hIjogImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IHsiZGV2aWNlTWFudWZhY3R1cmVyIjogIkFtc3RyYWQiLCAiYW5kcm9pZElkZmEiOiAic29tZV9hbmRyb2lkSWRmYSIsICJkZXZpY2VNb2RlbCI6ICJsYXJnZSIsICJvcGVuSWRmYSI6ICJzb21lX0lkZmEiLCAiY2FycmllciI6ICJzb21lX2NhcnJpZXIiLCAiYXBwbGVJZGZhIjogInNvbWVfYXBwbGVJZGZhIiwgIm9zVmVyc2lvbiI6ICIzLjAuMCIsICJhcHBsZUlkZnYiOiAic29tZV9hcHBsZUlkZnYiLCAib3NUeXBlIjogIk9TWCJ9fSwgeyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwgImRhdGEiOiB7ImxvbmdpdHVkZSI6IDEwLCAiYWx0aXR1ZGVBY2N1cmFjeSI6IDAuMywgImxhdGl0dWRlIjogNywgImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOiAwLjUsICJiZWFyaW5nIjogNTAsICJhbHRpdHVkZSI6IDIwLCAic3BlZWQiOiAxNn19XX0=", "se_ca": "my_category", "p": "pc"}, {"dtm": "1410184746895", "e": "se", "eid": "83eac227-7129-42a7-af45-40f63ed4b9dd", "se_ac": "another_action", "tv": "py-0.5.0", "cx": "eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY29udGV4dHMvanNvbnNjaGVtYS8xLTAtMCIsICJkYXRhIjogW3sic2NoZW1hIjogImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IHsiZGV2aWNlTWFudWZhY3R1cmVyIjogIkFtc3RyYWQiLCAiYW5kcm9pZElkZmEiOiAic29tZV9hbmRyb2lkSWRmYSIsICJkZXZpY2VNb2RlbCI6ICJsYXJnZSIsICJvcGVuSWRmYSI6ICJzb21lX0lkZmEiLCAiY2FycmllciI6ICJzb21lX2NhcnJpZXIiLCAiYXBwbGVJZGZhIjogInNvbWVfYXBwbGVJZGZhIiwgIm9zVmVyc2lvbiI6ICIzLjAuMCIsICJhcHBsZUlkZnYiOiAic29tZV9hcHBsZUlkZnYiLCAib3NUeXBlIjogIk9TWCJ9fSwgeyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwgImRhdGEiOiB7ImxvbmdpdHVkZSI6IDEwLCAiYWx0aXR1ZGVBY2N1cmFjeSI6IDAuMywgImxhdGl0dWRlIjogNywgImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOiAwLjUsICJiZWFyaW5nIjogNTAsICJhbHRpdHVkZSI6IDIwLCAic3BlZWQiOiAxNn19XX0=", "se_ca": "another_category", "p": "pc"}]}""".some ! DateTime - .parse("2014-09-08T13:59:07.000+00:00") ! + "Snowplow Tp2 POST w/ v0.6.0 collector" !! "2014-09-08 13:59:07 - - 37.157.33.123 POST - /com.snowplowanalytics.snowplow/tp2 200 - python-requests%2F2.2.1+CPython%2F3.3.5+Linux%2F3.2.0-61-generic &cv=clj-0.7.0-tom-0.1.0&nuid=5c6c40e4-eff8-409b-9327-471f303e30b6 - - - application%2Fjson%3B+charset%3Dutf-8 eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvcGF5bG9hZF9kYXRhL2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IFt7ImR0bSI6ICIxNDEwMTg0NzQ2ODk0IiwgImUiOiAicHYiLCAiZWlkIjogIjJjYWU0MTkxLTMxY2QtNDc4My04MmE4LWRmNTMxOGY0NGFmZiIsICJ1cmwiOiAiaHR0cDovL3d3dy5leGFtcGxlLmNvbSIsICJ0diI6ICJweS0wLjUuMCIsICJjeCI6ICJleUp6WTJobGJXRWlPaUFpYVdkc2RUcGpiMjB1YzI1dmQzQnNiM2RoYm1Gc2VYUnBZM011YzI1dmQzQnNiM2N2WTI5dWRHVjRkSE12YW5OdmJuTmphR1Z0WVM4eExUQXRNQ0lzSUNKa1lYUmhJam9nVzNzaWMyTm9aVzFoSWpvZ0ltbG5iSFU2WTI5dExuTnViM2R3Ykc5M1lXNWhiSGwwYVdOekxuTnViM2R3Ykc5M0wyMXZZbWxzWlY5amIyNTBaWGgwTDJwemIyNXpZMmhsYldFdk1TMHdMVEFpTENBaVpHRjBZU0k2SUhzaVpHVjJhV05sVFdGdWRXWmhZM1IxY21WeUlqb2dJa0Z0YzNSeVlXUWlMQ0FpWVc1a2NtOXBaRWxrWm1FaU9pQWljMjl0WlY5aGJtUnliMmxrU1dSbVlTSXNJQ0prWlhacFkyVk5iMlJsYkNJNklDSnNZWEpuWlNJc0lDSnZjR1Z1U1dSbVlTSTZJQ0p6YjIxbFgwbGtabUVpTENBaVkyRnljbWxsY2lJNklDSnpiMjFsWDJOaGNuSnBaWElpTENBaVlYQndiR1ZKWkdaaElqb2dJbk52YldWZllYQndiR1ZKWkdaaElpd2dJbTl6Vm1WeWMybHZiaUk2SUNJekxqQXVNQ0lzSUNKaGNIQnNaVWxrWm5ZaU9pQWljMjl0WlY5aGNIQnNaVWxrWm5ZaUxDQWliM05VZVhCbElqb2dJazlUV0NKOWZTd2dleUp6WTJobGJXRWlPaUFpYVdkc2RUcGpiMjB1YzI1dmQzQnNiM2RoYm1Gc2VYUnBZM011YzI1dmQzQnNiM2N2WjJWdmJHOWpZWFJwYjI1ZlkyOXVkR1Y0ZEM5cWMyOXVjMk5vWlcxaEx6RXRNQzB3SWl3Z0ltUmhkR0VpT2lCN0lteHZibWRwZEhWa1pTSTZJREV3TENBaVlXeDBhWFIxWkdWQlkyTjFjbUZqZVNJNklEQXVNeXdnSW14aGRHbDBkV1JsSWpvZ055d2dJbXhoZEdsMGRXUmxURzl1WjJsMGRXUmxRV05qZFhKaFkza2lPaUF3TGpVc0lDSmlaV0Z5YVc1bklqb2dOVEFzSUNKaGJIUnBkSFZrWlNJNklESXdMQ0FpYzNCbFpXUWlPaUF4Tm4xOVhYMD0iLCAicCI6ICJwYyJ9LCB7ImR0bSI6ICIxNDEwMTg0NzQ2ODk0IiwgImUiOiAic2UiLCAiZWlkIjogIjVhNzExODg1LTY5ZGMtNGY0Mi04Nzg1LWZjNjVmMTc1OGVjMCIsICJzZV9hYyI6ICJteV9hY3Rpb24iLCAidHYiOiAicHktMC41LjAiLCAiY3giOiAiZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdlkyOXVkR1Y0ZEhNdmFuTnZibk5qYUdWdFlTOHhMVEF0TUNJc0lDSmtZWFJoSWpvZ1czc2ljMk5vWlcxaElqb2dJbWxuYkhVNlkyOXRMbk51YjNkd2JHOTNZVzVoYkhsMGFXTnpMbk51YjNkd2JHOTNMMjF2WW1sc1pWOWpiMjUwWlhoMEwycHpiMjV6WTJobGJXRXZNUzB3TFRBaUxDQWlaR0YwWVNJNklIc2laR1YyYVdObFRXRnVkV1poWTNSMWNtVnlJam9nSWtGdGMzUnlZV1FpTENBaVlXNWtjbTlwWkVsa1ptRWlPaUFpYzI5dFpWOWhibVJ5YjJsa1NXUm1ZU0lzSUNKa1pYWnBZMlZOYjJSbGJDSTZJQ0pzWVhKblpTSXNJQ0p2Y0dWdVNXUm1ZU0k2SUNKemIyMWxYMGxrWm1FaUxDQWlZMkZ5Y21sbGNpSTZJQ0p6YjIxbFgyTmhjbkpwWlhJaUxDQWlZWEJ3YkdWSlpHWmhJam9nSW5OdmJXVmZZWEJ3YkdWSlpHWmhJaXdnSW05elZtVnljMmx2YmlJNklDSXpMakF1TUNJc0lDSmhjSEJzWlVsa1puWWlPaUFpYzI5dFpWOWhjSEJzWlVsa1puWWlMQ0FpYjNOVWVYQmxJam9nSWs5VFdDSjlmU3dnZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdloyVnZiRzlqWVhScGIyNWZZMjl1ZEdWNGRDOXFjMjl1YzJOb1pXMWhMekV0TUMwd0lpd2dJbVJoZEdFaU9pQjdJbXh2Ym1kcGRIVmtaU0k2SURFd0xDQWlZV3gwYVhSMVpHVkJZMk4xY21GamVTSTZJREF1TXl3Z0lteGhkR2wwZFdSbElqb2dOeXdnSW14aGRHbDBkV1JsVEc5dVoybDBkV1JsUVdOamRYSmhZM2tpT2lBd0xqVXNJQ0ppWldGeWFXNW5Jam9nTlRBc0lDSmhiSFJwZEhWa1pTSTZJREl3TENBaWMzQmxaV1FpT2lBeE5uMTlYWDA9IiwgInNlX2NhIjogIm15X2NhdGVnb3J5IiwgInAiOiAicGMifSwgeyJkdG0iOiAiMTQxMDE4NDc0Njg5NSIsICJlIjogInNlIiwgImVpZCI6ICI4M2VhYzIyNy03MTI5LTQyYTctYWY0NS00MGY2M2VkNGI5ZGQiLCAic2VfYWMiOiAiYW5vdGhlcl9hY3Rpb24iLCAidHYiOiAicHktMC41LjAiLCAiY3giOiAiZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdlkyOXVkR1Y0ZEhNdmFuTnZibk5qYUdWdFlTOHhMVEF0TUNJc0lDSmtZWFJoSWpvZ1czc2ljMk5vWlcxaElqb2dJbWxuYkhVNlkyOXRMbk51YjNkd2JHOTNZVzVoYkhsMGFXTnpMbk51YjNkd2JHOTNMMjF2WW1sc1pWOWpiMjUwWlhoMEwycHpiMjV6WTJobGJXRXZNUzB3TFRBaUxDQWlaR0YwWVNJNklIc2laR1YyYVdObFRXRnVkV1poWTNSMWNtVnlJam9nSWtGdGMzUnlZV1FpTENBaVlXNWtjbTlwWkVsa1ptRWlPaUFpYzI5dFpWOWhibVJ5YjJsa1NXUm1ZU0lzSUNKa1pYWnBZMlZOYjJSbGJDSTZJQ0pzWVhKblpTSXNJQ0p2Y0dWdVNXUm1ZU0k2SUNKemIyMWxYMGxrWm1FaUxDQWlZMkZ5Y21sbGNpSTZJQ0p6YjIxbFgyTmhjbkpwWlhJaUxDQWlZWEJ3YkdWSlpHWmhJam9nSW5OdmJXVmZZWEJ3YkdWSlpHWmhJaXdnSW05elZtVnljMmx2YmlJNklDSXpMakF1TUNJc0lDSmhjSEJzWlVsa1puWWlPaUFpYzI5dFpWOWhjSEJzWlVsa1puWWlMQ0FpYjNOVWVYQmxJam9nSWs5VFdDSjlmU3dnZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdloyVnZiRzlqWVhScGIyNWZZMjl1ZEdWNGRDOXFjMjl1YzJOb1pXMWhMekV0TUMwd0lpd2dJbVJoZEdFaU9pQjdJbXh2Ym1kcGRIVmtaU0k2SURFd0xDQWlZV3gwYVhSMVpHVkJZMk4xY21GamVTSTZJREF1TXl3Z0lteGhkR2wwZFdSbElqb2dOeXdnSW14aGRHbDBkV1JsVEc5dVoybDBkV1JsUVdOamRYSmhZM2tpT2lBd0xqVXNJQ0ppWldGeWFXNW5Jam9nTlRBc0lDSmhiSFJwZEhWa1pTSTZJREl3TENBaWMzQmxaV1FpT2lBeE5uMTlYWDA9IiwgInNlX2NhIjogImFub3RoZXJfY2F0ZWdvcnkiLCAicCI6ICJwYyJ9XX0" ! + "tp2" ! toNameValuePairs( + "cv" -> "clj-0.7.0-tom-0.1.0", + "nuid" -> "5c6c40e4-eff8-409b-9327-471f303e30b6") ! "application/json; charset=utf-8".some ! """{"schema": "iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-0", "data": [{"dtm": "1410184746894", "e": "pv", "eid": "2cae4191-31cd-4783-82a8-df5318f44aff", "url": "http://www.example.com", "tv": "py-0.5.0", "cx": "eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY29udGV4dHMvanNvbnNjaGVtYS8xLTAtMCIsICJkYXRhIjogW3sic2NoZW1hIjogImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IHsiZGV2aWNlTWFudWZhY3R1cmVyIjogIkFtc3RyYWQiLCAiYW5kcm9pZElkZmEiOiAic29tZV9hbmRyb2lkSWRmYSIsICJkZXZpY2VNb2RlbCI6ICJsYXJnZSIsICJvcGVuSWRmYSI6ICJzb21lX0lkZmEiLCAiY2FycmllciI6ICJzb21lX2NhcnJpZXIiLCAiYXBwbGVJZGZhIjogInNvbWVfYXBwbGVJZGZhIiwgIm9zVmVyc2lvbiI6ICIzLjAuMCIsICJhcHBsZUlkZnYiOiAic29tZV9hcHBsZUlkZnYiLCAib3NUeXBlIjogIk9TWCJ9fSwgeyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwgImRhdGEiOiB7ImxvbmdpdHVkZSI6IDEwLCAiYWx0aXR1ZGVBY2N1cmFjeSI6IDAuMywgImxhdGl0dWRlIjogNywgImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOiAwLjUsICJiZWFyaW5nIjogNTAsICJhbHRpdHVkZSI6IDIwLCAic3BlZWQiOiAxNn19XX0=", "p": "pc"}, {"dtm": "1410184746894", "e": "se", "eid": "5a711885-69dc-4f42-8785-fc65f1758ec0", "se_ac": "my_action", "tv": "py-0.5.0", "cx": "eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY29udGV4dHMvanNvbnNjaGVtYS8xLTAtMCIsICJkYXRhIjogW3sic2NoZW1hIjogImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IHsiZGV2aWNlTWFudWZhY3R1cmVyIjogIkFtc3RyYWQiLCAiYW5kcm9pZElkZmEiOiAic29tZV9hbmRyb2lkSWRmYSIsICJkZXZpY2VNb2RlbCI6ICJsYXJnZSIsICJvcGVuSWRmYSI6ICJzb21lX0lkZmEiLCAiY2FycmllciI6ICJzb21lX2NhcnJpZXIiLCAiYXBwbGVJZGZhIjogInNvbWVfYXBwbGVJZGZhIiwgIm9zVmVyc2lvbiI6ICIzLjAuMCIsICJhcHBsZUlkZnYiOiAic29tZV9hcHBsZUlkZnYiLCAib3NUeXBlIjogIk9TWCJ9fSwgeyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwgImRhdGEiOiB7ImxvbmdpdHVkZSI6IDEwLCAiYWx0aXR1ZGVBY2N1cmFjeSI6IDAuMywgImxhdGl0dWRlIjogNywgImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOiAwLjUsICJiZWFyaW5nIjogNTAsICJhbHRpdHVkZSI6IDIwLCAic3BlZWQiOiAxNn19XX0=", "se_ca": "my_category", "p": "pc"}, {"dtm": "1410184746895", "e": "se", "eid": "83eac227-7129-42a7-af45-40f63ed4b9dd", "se_ac": "another_action", "tv": "py-0.5.0", "cx": "eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvY29udGV4dHMvanNvbnNjaGVtYS8xLTAtMCIsICJkYXRhIjogW3sic2NoZW1hIjogImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IHsiZGV2aWNlTWFudWZhY3R1cmVyIjogIkFtc3RyYWQiLCAiYW5kcm9pZElkZmEiOiAic29tZV9hbmRyb2lkSWRmYSIsICJkZXZpY2VNb2RlbCI6ICJsYXJnZSIsICJvcGVuSWRmYSI6ICJzb21lX0lkZmEiLCAiY2FycmllciI6ICJzb21lX2NhcnJpZXIiLCAiYXBwbGVJZGZhIjogInNvbWVfYXBwbGVJZGZhIiwgIm9zVmVyc2lvbiI6ICIzLjAuMCIsICJhcHBsZUlkZnYiOiAic29tZV9hcHBsZUlkZnYiLCAib3NUeXBlIjogIk9TWCJ9fSwgeyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwgImRhdGEiOiB7ImxvbmdpdHVkZSI6IDEwLCAiYWx0aXR1ZGVBY2N1cmFjeSI6IDAuMywgImxhdGl0dWRlIjogNywgImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOiAwLjUsICJiZWFyaW5nIjogNTAsICJhbHRpdHVkZSI6IDIwLCAic3BlZWQiOiAxNn19XX0=", "se_ca": "another_category", "p": "pc"}]}""".some ! DateTime + .parse("2014-09-08T13:59:07.000+00:00") ! "python-requests%2F2.2.1+CPython%2F3.3.5+Linux%2F3.2.0-61-generic".some ! None | - "CallRail-style POST w/o body, content-type" !! "2013-08-29 00:18:48 - 830 37.157.33.123 POST d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/analytics/index.html Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0 e=pv&page=Introduction%20-%20Snowplow%20Analytics%25&dtm=1377735557970&tid=567074&vp=1024x635&ds=1024x635&vid=1&duid=7969620089de36eb&p=web&tv=js-0.12.0&fp=308909339&aid=snowplowweb&lang=en-US&cs=UTF-8&tz=America%2FLos_Angeles&refr=http%3A%2F%2Fwww.metacrawler.com%2Fsearch%2Fweb%3Ffcoid%3D417%26fcop%3Dtopnav%26fpid%3D27%26q%3Dsnowplow%2Banalytics%26ql%3D&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1024x768&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fanalytics%2Findex.html - - -" ! + "CallRail-style POST w/o body, content-type" !! "2013-08-29 00:18:48 - 830 37.157.33.123 POST d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/analytics/index.html Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0 e=pv&page=Introduction%20-%20Snowplow%20Analytics%25&dtm=1377735557970&tid=567074&vp=1024x635&ds=1024x635&vid=1&duid=7969620089de36eb&p=web&tv=js-0.12.0&fp=308909339&aid=snowplowweb&lang=en-US&cs=UTF-8&tz=America%2FLos_Angeles&refr=http%3A%2F%2Fwww.metacrawler.com%2Fsearch%2Fweb%3Ffcoid%3D417%26fcop%3Dtopnav%26fpid%3D27%26q%3Dsnowplow%2Banalytics%26ql%3D&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1024x768&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fanalytics%2Findex.html - - -" ! "tp1" ! toNameValuePairs( - "e" -> "pv", - "page" -> "Introduction - Snowplow Analytics%", - "dtm" -> "1377735557970", - "tid" -> "567074", - "vp" -> "1024x635", - "ds" -> "1024x635", - "vid" -> "1", - "duid" -> "7969620089de36eb", - "p" -> "web", - "tv" -> "js-0.12.0", - "fp" -> "308909339", - "aid" -> "snowplowweb", - "lang" -> "en-US", - "cs" -> "UTF-8", - "tz" -> "America/Los_Angeles", - "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", - "f_pdf" -> "1", - "f_qt" -> "1", + "e" -> "pv", + "page" -> "Introduction - Snowplow Analytics%", + "dtm" -> "1377735557970", + "tid" -> "567074", + "vp" -> "1024x635", + "ds" -> "1024x635", + "vid" -> "1", + "duid" -> "7969620089de36eb", + "p" -> "web", + "tv" -> "js-0.12.0", + "fp" -> "308909339", + "aid" -> "snowplowweb", + "lang" -> "en-US", + "cs" -> "UTF-8", + "tz" -> "America/Los_Angeles", + "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", + "f_pdf" -> "1", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "1", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "1", + "f_wma" -> "1", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "1", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "1024x768", - "cd" -> "24", - "cookie" -> "1", - "url" -> "http://snowplowanalytics.com/analytics/index.html" - ) ! None ! None ! DateTime.parse("2013-08-29T00:18:48.000+00:00") ! + "f_ag" -> "0", + "res" -> "1024x768", + "cd" -> "24", + "cookie" -> "1", + "url" -> "http://snowplowanalytics.com/analytics/index.html" + ) ! None ! None ! DateTime.parse("2013-08-29T00:18:48.000+00:00") ! "Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0".some ! "http://snowplowanalytics.com/analytics/index.html".some | // This may not be a valid GET but we need to accept it because Lumia emits it (#2743, #489) "Snowplow Tp1 GET w/ content-type no body " !! "2013-08-29 00:18:48 - 830 37.157.33.123 GET d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/analytics/index.html Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0 e=pv&page=Introduction%20-%20Snowplow%20Analytics%25&dtm=1377735557970&tid=567074&vp=1024x635&ds=1024x635&vid=1&duid=7969620089de36eb&p=web&tv=js-0.12.0&fp=308909339&aid=snowplowweb&lang=en-US&cs=UTF-8&tz=America%2FLos_Angeles&refr=http%3A%2F%2Fwww.metacrawler.com%2Fsearch%2Fweb%3Ffcoid%3D417%26fcop%3Dtopnav%26fpid%3D27%26q%3Dsnowplow%2Banalytics%26ql%3D&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1024x768&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fanalytics%2Findex.html - - - application%2Fx-www-form-urlencoded%3B+charset%3Dutf-8 -" ! "tp1" ! toNameValuePairs( - "e" -> "pv", - "page" -> "Introduction - Snowplow Analytics%", - "dtm" -> "1377735557970", - "tid" -> "567074", - "vp" -> "1024x635", - "ds" -> "1024x635", - "vid" -> "1", - "duid" -> "7969620089de36eb", - "p" -> "web", - "tv" -> "js-0.12.0", - "fp" -> "308909339", - "aid" -> "snowplowweb", - "lang" -> "en-US", - "cs" -> "UTF-8", - "tz" -> "America/Los_Angeles", - "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", - "f_pdf" -> "1", - "f_qt" -> "1", + "e" -> "pv", + "page" -> "Introduction - Snowplow Analytics%", + "dtm" -> "1377735557970", + "tid" -> "567074", + "vp" -> "1024x635", + "ds" -> "1024x635", + "vid" -> "1", + "duid" -> "7969620089de36eb", + "p" -> "web", + "tv" -> "js-0.12.0", + "fp" -> "308909339", + "aid" -> "snowplowweb", + "lang" -> "en-US", + "cs" -> "UTF-8", + "tz" -> "America/Los_Angeles", + "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", + "f_pdf" -> "1", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "1", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "1", + "f_wma" -> "1", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "1", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "1024x768", - "cd" -> "24", - "cookie" -> "1", - "url" -> "http://snowplowanalytics.com/analytics/index.html" + "f_ag" -> "0", + "res" -> "1024x768", + "cd" -> "24", + "cookie" -> "1", + "url" -> "http://snowplowanalytics.com/analytics/index.html" ) ! "application/x-www-form-urlencoded; charset=utf-8".some ! None ! DateTime.parse( - "2013-08-29T00:18:48.000+00:00") ! + "2013-08-29T00:18:48.000+00:00") ! "Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0".some ! "http://snowplowanalytics.com/analytics/index.html".some |> { (_, raw, version, payload, contentType, body, timestamp, userAgent, refererUri) => @@ -186,12 +192,13 @@ class CljTomcatLoaderSpec extends Specification with DataTables with ValidationM .toCollectorPayload(raw) val expected = CollectorPayload( - api = CollectorApi(Expected.vendor, version), + api = CollectorApi(Expected.vendor, version), querystring = payload, contentType = contentType, - body = body, - source = CollectorSource(Expected.collector, Expected.encoding, None), - context = CollectorContext(timestamp.some, Expected.ipAddress, userAgent, refererUri, Nil, None) + body = body, + source = CollectorSource(Expected.collector, Expected.encoding, None), + context = + CollectorContext(timestamp.some, Expected.ipAddress, userAgent, refererUri, Nil, None) ) canonicalEvent must beSuccessful(expected.some) @@ -203,7 +210,8 @@ class CljTomcatLoaderSpec extends Specification with DataTables with ValidationM "2014-09-08 13:59:07 - - 37.157.33.123 GET - /com.snowplowanalytics.snowplow/tp2 200 - python-requests%2F2.2.1+CPython%2F3.3.5+Linux%2F3.2.0-61-generic &cv=clj-0.7.0-tom-0.1.0&nuid=5c6c40e4-eff8-409b-9327-471f303e30b6 - - - application%2Fjson%3B+charset%3Dutf-8 eyJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvcGF5bG9hZF9kYXRhL2pzb25zY2hlbWEvMS0wLTAiLCAiZGF0YSI6IFt7ImR0bSI6ICIxNDEwMTg0NzQ2ODk0IiwgImUiOiAicHYiLCAiZWlkIjogIjJjYWU0MTkxLTMxY2QtNDc4My04MmE4LWRmNTMxOGY0NGFmZiIsICJ1cmwiOiAiaHR0cDovL3d3dy5leGFtcGxlLmNvbSIsICJ0diI6ICJweS0wLjUuMCIsICJjeCI6ICJleUp6WTJobGJXRWlPaUFpYVdkc2RUcGpiMjB1YzI1dmQzQnNiM2RoYm1Gc2VYUnBZM011YzI1dmQzQnNiM2N2WTI5dWRHVjRkSE12YW5OdmJuTmphR1Z0WVM4eExUQXRNQ0lzSUNKa1lYUmhJam9nVzNzaWMyTm9aVzFoSWpvZ0ltbG5iSFU2WTI5dExuTnViM2R3Ykc5M1lXNWhiSGwwYVdOekxuTnViM2R3Ykc5M0wyMXZZbWxzWlY5amIyNTBaWGgwTDJwemIyNXpZMmhsYldFdk1TMHdMVEFpTENBaVpHRjBZU0k2SUhzaVpHVjJhV05sVFdGdWRXWmhZM1IxY21WeUlqb2dJa0Z0YzNSeVlXUWlMQ0FpWVc1a2NtOXBaRWxrWm1FaU9pQWljMjl0WlY5aGJtUnliMmxrU1dSbVlTSXNJQ0prWlhacFkyVk5iMlJsYkNJNklDSnNZWEpuWlNJc0lDSnZjR1Z1U1dSbVlTSTZJQ0p6YjIxbFgwbGtabUVpTENBaVkyRnljbWxsY2lJNklDSnpiMjFsWDJOaGNuSnBaWElpTENBaVlYQndiR1ZKWkdaaElqb2dJbk52YldWZllYQndiR1ZKWkdaaElpd2dJbTl6Vm1WeWMybHZiaUk2SUNJekxqQXVNQ0lzSUNKaGNIQnNaVWxrWm5ZaU9pQWljMjl0WlY5aGNIQnNaVWxrWm5ZaUxDQWliM05VZVhCbElqb2dJazlUV0NKOWZTd2dleUp6WTJobGJXRWlPaUFpYVdkc2RUcGpiMjB1YzI1dmQzQnNiM2RoYm1Gc2VYUnBZM011YzI1dmQzQnNiM2N2WjJWdmJHOWpZWFJwYjI1ZlkyOXVkR1Y0ZEM5cWMyOXVjMk5vWlcxaEx6RXRNQzB3SWl3Z0ltUmhkR0VpT2lCN0lteHZibWRwZEhWa1pTSTZJREV3TENBaVlXeDBhWFIxWkdWQlkyTjFjbUZqZVNJNklEQXVNeXdnSW14aGRHbDBkV1JsSWpvZ055d2dJbXhoZEdsMGRXUmxURzl1WjJsMGRXUmxRV05qZFhKaFkza2lPaUF3TGpVc0lDSmlaV0Z5YVc1bklqb2dOVEFzSUNKaGJIUnBkSFZrWlNJNklESXdMQ0FpYzNCbFpXUWlPaUF4Tm4xOVhYMD0iLCAicCI6ICJwYyJ9LCB7ImR0bSI6ICIxNDEwMTg0NzQ2ODk0IiwgImUiOiAic2UiLCAiZWlkIjogIjVhNzExODg1LTY5ZGMtNGY0Mi04Nzg1LWZjNjVmMTc1OGVjMCIsICJzZV9hYyI6ICJteV9hY3Rpb24iLCAidHYiOiAicHktMC41LjAiLCAiY3giOiAiZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdlkyOXVkR1Y0ZEhNdmFuTnZibk5qYUdWdFlTOHhMVEF0TUNJc0lDSmtZWFJoSWpvZ1czc2ljMk5vWlcxaElqb2dJbWxuYkhVNlkyOXRMbk51YjNkd2JHOTNZVzVoYkhsMGFXTnpMbk51YjNkd2JHOTNMMjF2WW1sc1pWOWpiMjUwWlhoMEwycHpiMjV6WTJobGJXRXZNUzB3TFRBaUxDQWlaR0YwWVNJNklIc2laR1YyYVdObFRXRnVkV1poWTNSMWNtVnlJam9nSWtGdGMzUnlZV1FpTENBaVlXNWtjbTlwWkVsa1ptRWlPaUFpYzI5dFpWOWhibVJ5YjJsa1NXUm1ZU0lzSUNKa1pYWnBZMlZOYjJSbGJDSTZJQ0pzWVhKblpTSXNJQ0p2Y0dWdVNXUm1ZU0k2SUNKemIyMWxYMGxrWm1FaUxDQWlZMkZ5Y21sbGNpSTZJQ0p6YjIxbFgyTmhjbkpwWlhJaUxDQWlZWEJ3YkdWSlpHWmhJam9nSW5OdmJXVmZZWEJ3YkdWSlpHWmhJaXdnSW05elZtVnljMmx2YmlJNklDSXpMakF1TUNJc0lDSmhjSEJzWlVsa1puWWlPaUFpYzI5dFpWOWhjSEJzWlVsa1puWWlMQ0FpYjNOVWVYQmxJam9nSWs5VFdDSjlmU3dnZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdloyVnZiRzlqWVhScGIyNWZZMjl1ZEdWNGRDOXFjMjl1YzJOb1pXMWhMekV0TUMwd0lpd2dJbVJoZEdFaU9pQjdJbXh2Ym1kcGRIVmtaU0k2SURFd0xDQWlZV3gwYVhSMVpHVkJZMk4xY21GamVTSTZJREF1TXl3Z0lteGhkR2wwZFdSbElqb2dOeXdnSW14aGRHbDBkV1JsVEc5dVoybDBkV1JsUVdOamRYSmhZM2tpT2lBd0xqVXNJQ0ppWldGeWFXNW5Jam9nTlRBc0lDSmhiSFJwZEhWa1pTSTZJREl3TENBaWMzQmxaV1FpT2lBeE5uMTlYWDA9IiwgInNlX2NhIjogIm15X2NhdGVnb3J5IiwgInAiOiAicGMifSwgeyJkdG0iOiAiMTQxMDE4NDc0Njg5NSIsICJlIjogInNlIiwgImVpZCI6ICI4M2VhYzIyNy03MTI5LTQyYTctYWY0NS00MGY2M2VkNGI5ZGQiLCAic2VfYWMiOiAiYW5vdGhlcl9hY3Rpb24iLCAidHYiOiAicHktMC41LjAiLCAiY3giOiAiZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdlkyOXVkR1Y0ZEhNdmFuTnZibk5qYUdWdFlTOHhMVEF0TUNJc0lDSmtZWFJoSWpvZ1czc2ljMk5vWlcxaElqb2dJbWxuYkhVNlkyOXRMbk51YjNkd2JHOTNZVzVoYkhsMGFXTnpMbk51YjNkd2JHOTNMMjF2WW1sc1pWOWpiMjUwWlhoMEwycHpiMjV6WTJobGJXRXZNUzB3TFRBaUxDQWlaR0YwWVNJNklIc2laR1YyYVdObFRXRnVkV1poWTNSMWNtVnlJam9nSWtGdGMzUnlZV1FpTENBaVlXNWtjbTlwWkVsa1ptRWlPaUFpYzI5dFpWOWhibVJ5YjJsa1NXUm1ZU0lzSUNKa1pYWnBZMlZOYjJSbGJDSTZJQ0pzWVhKblpTSXNJQ0p2Y0dWdVNXUm1ZU0k2SUNKemIyMWxYMGxrWm1FaUxDQWlZMkZ5Y21sbGNpSTZJQ0p6YjIxbFgyTmhjbkpwWlhJaUxDQWlZWEJ3YkdWSlpHWmhJam9nSW5OdmJXVmZZWEJ3YkdWSlpHWmhJaXdnSW05elZtVnljMmx2YmlJNklDSXpMakF1TUNJc0lDSmhjSEJzWlVsa1puWWlPaUFpYzI5dFpWOWhjSEJzWlVsa1puWWlMQ0FpYjNOVWVYQmxJam9nSWs5VFdDSjlmU3dnZXlKelkyaGxiV0VpT2lBaWFXZHNkVHBqYjIwdWMyNXZkM0JzYjNkaGJtRnNlWFJwWTNNdWMyNXZkM0JzYjNjdloyVnZiRzlqWVhScGIyNWZZMjl1ZEdWNGRDOXFjMjl1YzJOb1pXMWhMekV0TUMwd0lpd2dJbVJoZEdFaU9pQjdJbXh2Ym1kcGRIVmtaU0k2SURFd0xDQWlZV3gwYVhSMVpHVkJZMk4xY21GamVTSTZJREF1TXl3Z0lteGhkR2wwZFdSbElqb2dOeXdnSW14aGRHbDBkV1JsVEc5dVoybDBkV1JsUVdOamRYSmhZM2tpT2lBd0xqVXNJQ0ppWldGeWFXNW5Jam9nTlRBc0lDSmhiSFJwZEhWa1pTSTZJREl3TENBaWMzQmxaV1FpT2lBeE5uMTlYWDA9IiwgInNlX2NhIjogImFub3RoZXJfY2F0ZWdvcnkiLCAicCI6ICJwYyJ9XX0" val actual = CljTomcatLoader.toCollectorPayload(raw) actual must beFailing( - NonEmptyList("Operation must be POST, not GET, if request content type and/or body are provided")) + NonEmptyList( + "Operation must be POST, not GET, if request content type and/or body are provided")) } def e3 = { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CloudfrontLoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CloudfrontLoaderSpec.scala index eaca11367..08c9b6f43 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CloudfrontLoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CloudfrontLoaderSpec.scala @@ -24,7 +24,11 @@ import Scalaz._ import utils.ConversionUtils import SpecHelpers._ -class CloudfrontLoaderSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class CloudfrontLoaderSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the CloudfrontLoader functionality toTimestamp should create a DateTime from valid date and time Strings $e1 @@ -38,21 +42,24 @@ class CloudfrontLoaderSpec extends Specification with DataTables with Validation object Expected { val collector = "cloudfront" - val encoding = "UTF-8" - val api = CollectorApi("com.snowplowanalytics.snowplow", "tp1") + val encoding = "UTF-8" + val api = CollectorApi("com.snowplowanalytics.snowplow", "tp1") } def e1 = - "SPEC NAME" || "DATE" | "TIME" | "EXP. DATETIME" | - "Valid with ms #1" !! "2003-12-04" ! "00:18:48.234" ! DateTime.parse("2003-12-04T00:18:48.234+00:00") | - "Valid with ms #2" !! "2011-08-29" ! "23:56:01.003" ! DateTime.parse("2011-08-29T23:56:01.003+00:00") | - "Valid without ms #1" !! "2013-05-12" ! "17:34:10" ! DateTime.parse("2013-05-12T17:34:10+00:00") | - "Valid without ms #2" !! "1980-04-01" ! "21:20:04" ! DateTime.parse("1980-04-01T21:20:04+00:00") |> { - (_, date, time, expected) => - { - val actual = CloudfrontLoader.toTimestamp(date, time) - actual must beSuccessful(expected) - } + "SPEC NAME" || "DATE" | "TIME" | "EXP. DATETIME" | + "Valid with ms #1" !! "2003-12-04" ! "00:18:48.234" ! DateTime.parse( + "2003-12-04T00:18:48.234+00:00") | + "Valid with ms #2" !! "2011-08-29" ! "23:56:01.003" ! DateTime.parse( + "2011-08-29T23:56:01.003+00:00") | + "Valid without ms #1" !! "2013-05-12" ! "17:34:10" ! DateTime.parse( + "2013-05-12T17:34:10+00:00") | + "Valid without ms #2" !! "1980-04-01" ! "21:20:04" ! DateTime.parse( + "1980-04-01T21:20:04+00:00") |> { (_, date, time, expected) => + { + val actual = CloudfrontLoader.toTimestamp(date, time) + actual must beSuccessful(expected) + } } def e2 = @@ -61,23 +68,24 @@ class CloudfrontLoaderSpec extends Specification with DataTables with Validation } def e3 = - "SPEC NAME" || "URI" | "EXP. URI" | - "URI with trailing % #1" !! "https://github.com/snowplow/snowplow/issues/494%" ! "https://github.com/snowplow/snowplow/issues/494" | - "URI with trailing % #2" !! "http://bbc.co.uk%" ! "http://bbc.co.uk" | - "URI without trailing % #1" !! "https://github.com/snowplow/snowplow/issues/494" ! "https://github.com/snowplow/snowplow/issues/494" | - "URI without trailing % #2" !! "http://bbc.co.uk" ! "http://bbc.co.uk" |> { (_, uri, expected) => - { - val actual = CloudfrontLoader.toCleanUri(uri) - actual must_== expected - } + "SPEC NAME" || "URI" | "EXP. URI" | + "URI with trailing % #1" !! "https://github.com/snowplow/snowplow/issues/494%" ! "https://github.com/snowplow/snowplow/issues/494" | + "URI with trailing % #2" !! "http://bbc.co.uk%" ! "http://bbc.co.uk" | + "URI without trailing % #1" !! "https://github.com/snowplow/snowplow/issues/494" ! "https://github.com/snowplow/snowplow/issues/494" | + "URI without trailing % #2" !! "http://bbc.co.uk" ! "http://bbc.co.uk" |> { + (_, uri, expected) => + { + val actual = CloudfrontLoader.toCleanUri(uri) + actual must_== expected + } } def e4 = - "SPEC NAME" || "QUERYSTRING" | "EXP. QUERYSTRING" | - "Double-encoded %s, modify" !! "e=pv&page=Celestial%2520Tarot%2520-%2520Psychic%2520Bazaar&dtm=1376487150616&tid=483686&vp=1097x482&ds=1097x1973&vid=1&duid=1f2719e9217b5e1b&p=web&tv=js-0.12.0&fp=3748874661&aid=pbzsite&lang=en-IE&cs=utf-8&tz=Europe%252FLondon&refr=http%253A%252F%252Fwww.psychicbazaar.com%252Fsearch%253Fsearch_query%253Dcelestial%252Btarot%252Bdeck&f_java=1&res=1097x617&cd=24&cookie=1&url=http%253A%252F%252Fwww.psychicbazaar.com%252Ftarot-cards%252F48-celestial-tarot.html" ! "e=pv&page=Celestial%20Tarot%20-%20Psychic%20Bazaar&dtm=1376487150616&tid=483686&vp=1097x482&ds=1097x1973&vid=1&duid=1f2719e9217b5e1b&p=web&tv=js-0.12.0&fp=3748874661&aid=pbzsite&lang=en-IE&cs=utf-8&tz=Europe%2FLondon&refr=http%3A%2F%2Fwww.psychicbazaar.com%2Fsearch%3Fsearch_query%3Dcelestial%2Btarot%2Bdeck&f_java=1&res=1097x617&cd=24&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2Ftarot-cards%2F48-celestial-tarot.html" | - "Ambiguous - assume double-encoded, modify" !! "%2588 is 1x-encoded 25 percent OR 2x-encoded ^" ! "%88 is 1x-encoded 25 percent OR 2x-encoded ^" | - "Single-encoded %s, leave" !! "e=pp&page=Dreaming%20Way%20Tarot%20-%20Psychic%20Bazaar&pp_mix=0&pp_max=0&pp_miy=0&pp_may=0&dtm=1376984181667&tid=056188&vp=1440x838&ds=1440x1401&vid=1&duid=8ac2d67163d6d36a&p=web&tv=js-0.12.0&fp=1569742263&aid=pbzsite&lang=en-us&cs=UTF-8&tz=Australia%2FSydney&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1440x900&cd=24&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2Ftarot-cards%2F312-dreaming-way-tarot.html" ! "e=pp&page=Dreaming%20Way%20Tarot%20-%20Psychic%20Bazaar&pp_mix=0&pp_max=0&pp_miy=0&pp_may=0&dtm=1376984181667&tid=056188&vp=1440x838&ds=1440x1401&vid=1&duid=8ac2d67163d6d36a&p=web&tv=js-0.12.0&fp=1569742263&aid=pbzsite&lang=en-us&cs=UTF-8&tz=Australia%2FSydney&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1440x900&cd=24&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2Ftarot-cards%2F312-dreaming-way-tarot.html" | - "Single-encoded % sign itself, leave" !! "Loading - 70%25 Complete" ! "Loading - 70%25 Complete" |> { + "SPEC NAME" || "QUERYSTRING" | "EXP. QUERYSTRING" | + "Double-encoded %s, modify" !! "e=pv&page=Celestial%2520Tarot%2520-%2520Psychic%2520Bazaar&dtm=1376487150616&tid=483686&vp=1097x482&ds=1097x1973&vid=1&duid=1f2719e9217b5e1b&p=web&tv=js-0.12.0&fp=3748874661&aid=pbzsite&lang=en-IE&cs=utf-8&tz=Europe%252FLondon&refr=http%253A%252F%252Fwww.psychicbazaar.com%252Fsearch%253Fsearch_query%253Dcelestial%252Btarot%252Bdeck&f_java=1&res=1097x617&cd=24&cookie=1&url=http%253A%252F%252Fwww.psychicbazaar.com%252Ftarot-cards%252F48-celestial-tarot.html" ! "e=pv&page=Celestial%20Tarot%20-%20Psychic%20Bazaar&dtm=1376487150616&tid=483686&vp=1097x482&ds=1097x1973&vid=1&duid=1f2719e9217b5e1b&p=web&tv=js-0.12.0&fp=3748874661&aid=pbzsite&lang=en-IE&cs=utf-8&tz=Europe%2FLondon&refr=http%3A%2F%2Fwww.psychicbazaar.com%2Fsearch%3Fsearch_query%3Dcelestial%2Btarot%2Bdeck&f_java=1&res=1097x617&cd=24&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2Ftarot-cards%2F48-celestial-tarot.html" | + "Ambiguous - assume double-encoded, modify" !! "%2588 is 1x-encoded 25 percent OR 2x-encoded ^" ! "%88 is 1x-encoded 25 percent OR 2x-encoded ^" | + "Single-encoded %s, leave" !! "e=pp&page=Dreaming%20Way%20Tarot%20-%20Psychic%20Bazaar&pp_mix=0&pp_max=0&pp_miy=0&pp_may=0&dtm=1376984181667&tid=056188&vp=1440x838&ds=1440x1401&vid=1&duid=8ac2d67163d6d36a&p=web&tv=js-0.12.0&fp=1569742263&aid=pbzsite&lang=en-us&cs=UTF-8&tz=Australia%2FSydney&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1440x900&cd=24&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2Ftarot-cards%2F312-dreaming-way-tarot.html" ! "e=pp&page=Dreaming%20Way%20Tarot%20-%20Psychic%20Bazaar&pp_mix=0&pp_max=0&pp_miy=0&pp_may=0&dtm=1376984181667&tid=056188&vp=1440x838&ds=1440x1401&vid=1&duid=8ac2d67163d6d36a&p=web&tv=js-0.12.0&fp=1569742263&aid=pbzsite&lang=en-us&cs=UTF-8&tz=Australia%2FSydney&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1440x900&cd=24&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2Ftarot-cards%2F312-dreaming-way-tarot.html" | + "Single-encoded % sign itself, leave" !! "Loading - 70%25 Complete" ! "Loading - 70%25 Complete" |> { (_, qs, expected) => { val actual = ConversionUtils.singleEncodePcts(qs) @@ -86,131 +94,131 @@ class CloudfrontLoaderSpec extends Specification with DataTables with Validation } def e5 = - "SPEC NAME" || "RAW" | "EXP. TIMESTAMP" | "EXP. PAYLOAD" | "EXP. IP ADDRESS" | "EXP. USER AGENT" | "EXP. REFERER URI" | + "SPEC NAME" || "RAW" | "EXP. TIMESTAMP" | "EXP. PAYLOAD" | "EXP. IP ADDRESS" | "EXP. USER AGENT" | "EXP. REFERER URI" | "CloudFront with 2 spaces" !! "2013-08-29 00:18:48 LAX3 830 255.255.255.255 GET d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/analytics/index.html Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0 e=pv&page=Introduction%20-%20Snowplow%20Analytics%25&dtm=1377735557970&tid=567074&vp=1024x635&ds=1024x635&vid=1&duid=7969620089de36eb&p=web&tv=js-0.12.0&fp=308909339&aid=snowplowweb&lang=en-US&cs=UTF-8&tz=America%2FLos_Angeles&refr=http%3A%2F%2Fwww.metacrawler.com%2Fsearch%2Fweb%3Ffcoid%3D417%26fcop%3Dtopnav%26fpid%3D27%26q%3Dsnowplow%2Banalytics%26ql%3D&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1024x768&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fanalytics%2Findex.html - Hit wQ1OBZtQlGgfM_tPEJ-lIQLsdra0U-lXgmfJfwja2KAV_SfTdT3lZg==" ! DateTime.parse("2013-08-29T00:18:48.000+00:00") ! toNameValuePairs( - "e" -> "pv", - "page" -> "Introduction - Snowplow Analytics%", - "dtm" -> "1377735557970", - "tid" -> "567074", - "vp" -> "1024x635", - "ds" -> "1024x635", - "vid" -> "1", - "duid" -> "7969620089de36eb", - "p" -> "web", - "tv" -> "js-0.12.0", - "fp" -> "308909339", - "aid" -> "snowplowweb", - "lang" -> "en-US", - "cs" -> "UTF-8", - "tz" -> "America/Los_Angeles", - "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", - "f_pdf" -> "1", - "f_qt" -> "1", + "e" -> "pv", + "page" -> "Introduction - Snowplow Analytics%", + "dtm" -> "1377735557970", + "tid" -> "567074", + "vp" -> "1024x635", + "ds" -> "1024x635", + "vid" -> "1", + "duid" -> "7969620089de36eb", + "p" -> "web", + "tv" -> "js-0.12.0", + "fp" -> "308909339", + "aid" -> "snowplowweb", + "lang" -> "en-US", + "cs" -> "UTF-8", + "tz" -> "America/Los_Angeles", + "refr" -> "http://www.metacrawler.com/search/web?fcoid=417&fcop=topnav&fpid=27&q=snowplow+analytics&ql=", + "f_pdf" -> "1", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "1", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "1", + "f_wma" -> "1", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "1", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "1024x768", - "cd" -> "24", - "cookie" -> "1", - "url" -> "http://snowplowanalytics.com/analytics/index.html" - ) ! - "255.255.255.255".some ! "Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0".some ! "http://snowplowanalytics.com/analytics/index.html".some | + "f_ag" -> "0", + "res" -> "1024x768", + "cd" -> "24", + "cookie" -> "1", + "url" -> "http://snowplowanalytics.com/analytics/index.html" + ) ! + "255.255.255.255".some ! "Mozilla/5.0%20(Windows%20NT%205.1;%20rv:23.0)%20Gecko/20100101%20Firefox/23.0".some ! "http://snowplowanalytics.com/analytics/index.html".some | "CloudFront with 4 spaces" !! "2014-01-28 02:52:24 HKG50 829 202.134.75.113 GET d3v6ndkyapxc2w.cloudfront.net /i 200 http://snowplowanalytics.com/product/index.html Mozilla/5.0%2520(Windows%2520NT%25205.1)%2520AppleWebKit/537.36%2520(KHTML,%2520like%2520Gecko)%2520Chrome/31.0.1650.57%2520Safari/537.36 e=pv&page=Snowplow%2520-%2520the%2520most%2520powerful%252C%2520scalable%252C%2520flexible%2520web%2520analytics%2520platform%2520in%2520the%2520world.%2520-%2520Snowplow%2520Analytics&tid=322602&vp=1600x739&ds=1600x739&vid=1&duid=5c34698b211e8949&p=web&tv=js-0.13.0&aid=snowplowweb&lang=zh-CN&cs=UTF-8&tz=Asia%252FShanghai&refr=http%253A%252F%252Fsnowplowanalytics.com%252Fabout%252Findex.html&f_pdf=1&f_qt=1&f_realp=0&f_wma=1&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=1600x900&cookie=1&url=http%253A%252F%252Fsnowplowanalytics.com%252Fproduct%252Findex.html - Hit VtgzUTq1UoySDN3m_B-5DqmpTjgAS5YaAcvk_uz_D0-0TrDrZJJu2Q== d3v6ndkyapxc2w.cloudfront.net http 881" ! DateTime.parse("2014-01-28T02:52:24.000+00:00") ! toNameValuePairs( - "e" -> "pv", - "page" -> "Snowplow - the most powerful, scalable, flexible web analytics platform in the world. - Snowplow Analytics", - "tid" -> "322602", - "vp" -> "1600x739", - "ds" -> "1600x739", - "vid" -> "1", - "duid" -> "5c34698b211e8949", - "p" -> "web", - "tv" -> "js-0.13.0", - "aid" -> "snowplowweb", - "lang" -> "zh-CN", - "cs" -> "UTF-8", - "tz" -> "Asia/Shanghai", - "refr" -> "http://snowplowanalytics.com/about/index.html", - "f_pdf" -> "1", - "f_qt" -> "1", + "e" -> "pv", + "page" -> "Snowplow - the most powerful, scalable, flexible web analytics platform in the world. - Snowplow Analytics", + "tid" -> "322602", + "vp" -> "1600x739", + "ds" -> "1600x739", + "vid" -> "1", + "duid" -> "5c34698b211e8949", + "p" -> "web", + "tv" -> "js-0.13.0", + "aid" -> "snowplowweb", + "lang" -> "zh-CN", + "cs" -> "UTF-8", + "tz" -> "Asia/Shanghai", + "refr" -> "http://snowplowanalytics.com/about/index.html", + "f_pdf" -> "1", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "1", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "1", + "f_wma" -> "1", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "1", "f_gears" -> "0", - "f_ag" -> "1", - "res" -> "1600x900", - "cookie" -> "1", - "url" -> "http://snowplowanalytics.com/product/index.html" - ) ! + "f_ag" -> "1", + "res" -> "1600x900", + "cookie" -> "1", + "url" -> "http://snowplowanalytics.com/product/index.html" + ) ! "202.134.75.113".some ! "Mozilla/5.0%20(Windows%20NT%205.1)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/31.0.1650.57%20Safari/537.36".some ! "http://snowplowanalytics.com/product/index.html".some | - "CloudFront with tabs" !! "2014-01-28 03:41:59 IAD12 828 67.71.16.237 GET d10wr4jwvp55f9.cloudfront.net /i 200 http://www.psychicbazaar.com/oracles/107-magdalene-oracle.html Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520Trident/7.0;%2520rv:11.0)%2520like%2520Gecko e=pp&page=Magdalene%2520Oracle%2520-%2520Psychic%2520Bazaar&tid=151507&vp=975x460&ds=1063x1760&vid=1&duid=44a32544aac965f4&p=web&tv=js-0.13.0&aid=pbzsite&lang=en-CA&cs=utf-8&tz=America%252FHavana&refr=http%253A%252F%252Fwww.google.ca%252Furl%253Fsa%253Dt%2526rct%253Dj%2526q%253D%2526esrc%253Ds%2526source%253Dweb%2526cd%253D16%2526ved%253D0CIIBEBYwDw%2526url%253Dhttp%25253A%25252F%25252Fwww.psychicbazaar.com%25252Foracles%25252F107-magdalene-oracle.html%2526ei%253DIibnUsfBDMiM2gXGoICoDg%2526usg%253DAFQjCNE6fEqO8lnxDHeke0LOuAZIa1iSFQ%2526sig2%253DV7KJR0VmGw5yaHoMKKJHhg%2526bvm%253Dbv.59930103%252Cd.b2I&f_pdf=0&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=0&f_java=1&f_gears=0&f_ag=1&res=975x571&cookie=1&url=http%253A%252F%252Fwww.psychicbazaar.com%252Foracles%252F107-magdalene-oracle.html - Hit 7T7tuHtEcdoDvUuGnQ3F0RI_UEWOUeb0b-YIhcoxjziuEBMDcKv_OA== d10wr4jwvp55f9.cloudfront.net http 1047" ! + "CloudFront with tabs" !! "2014-01-28 03:41:59 IAD12 828 67.71.16.237 GET d10wr4jwvp55f9.cloudfront.net /i 200 http://www.psychicbazaar.com/oracles/107-magdalene-oracle.html Mozilla/5.0%2520(Windows%2520NT%25206.1;%2520Trident/7.0;%2520rv:11.0)%2520like%2520Gecko e=pp&page=Magdalene%2520Oracle%2520-%2520Psychic%2520Bazaar&tid=151507&vp=975x460&ds=1063x1760&vid=1&duid=44a32544aac965f4&p=web&tv=js-0.13.0&aid=pbzsite&lang=en-CA&cs=utf-8&tz=America%252FHavana&refr=http%253A%252F%252Fwww.google.ca%252Furl%253Fsa%253Dt%2526rct%253Dj%2526q%253D%2526esrc%253Ds%2526source%253Dweb%2526cd%253D16%2526ved%253D0CIIBEBYwDw%2526url%253Dhttp%25253A%25252F%25252Fwww.psychicbazaar.com%25252Foracles%25252F107-magdalene-oracle.html%2526ei%253DIibnUsfBDMiM2gXGoICoDg%2526usg%253DAFQjCNE6fEqO8lnxDHeke0LOuAZIa1iSFQ%2526sig2%253DV7KJR0VmGw5yaHoMKKJHhg%2526bvm%253Dbv.59930103%252Cd.b2I&f_pdf=0&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=0&f_java=1&f_gears=0&f_ag=1&res=975x571&cookie=1&url=http%253A%252F%252Fwww.psychicbazaar.com%252Foracles%252F107-magdalene-oracle.html - Hit 7T7tuHtEcdoDvUuGnQ3F0RI_UEWOUeb0b-YIhcoxjziuEBMDcKv_OA== d10wr4jwvp55f9.cloudfront.net http 1047" ! DateTime.parse("2014-01-28T03:41:59.000+00:00") ! toNameValuePairs( - "e" -> "pp", - "page" -> "Magdalene Oracle - Psychic Bazaar", - "tid" -> "151507", - "vp" -> "975x460", - "ds" -> "1063x1760", - "vid" -> "1", - "duid" -> "44a32544aac965f4", - "p" -> "web", - "tv" -> "js-0.13.0", - "aid" -> "pbzsite", - "lang" -> "en-CA", - "cs" -> "utf-8", - "tz" -> "America/Havana", - "refr" -> "http://www.google.ca/url?sa=t&rct=j&q=&esrc=s&source=web&cd=16&ved=0CIIBEBYwDw&url=http%3A%2F%2Fwww.psychicbazaar.com%2Foracles%2F107-magdalene-oracle.html&ei=IibnUsfBDMiM2gXGoICoDg&usg=AFQjCNE6fEqO8lnxDHeke0LOuAZIa1iSFQ&sig2=V7KJR0VmGw5yaHoMKKJHhg&bvm=bv.59930103,d.b2I", - "f_pdf" -> "0", - "f_qt" -> "0", + "e" -> "pp", + "page" -> "Magdalene Oracle - Psychic Bazaar", + "tid" -> "151507", + "vp" -> "975x460", + "ds" -> "1063x1760", + "vid" -> "1", + "duid" -> "44a32544aac965f4", + "p" -> "web", + "tv" -> "js-0.13.0", + "aid" -> "pbzsite", + "lang" -> "en-CA", + "cs" -> "utf-8", + "tz" -> "America/Havana", + "refr" -> "http://www.google.ca/url?sa=t&rct=j&q=&esrc=s&source=web&cd=16&ved=0CIIBEBYwDw&url=http%3A%2F%2Fwww.psychicbazaar.com%2Foracles%2F107-magdalene-oracle.html&ei=IibnUsfBDMiM2gXGoICoDg&usg=AFQjCNE6fEqO8lnxDHeke0LOuAZIa1iSFQ&sig2=V7KJR0VmGw5yaHoMKKJHhg&bvm=bv.59930103,d.b2I", + "f_pdf" -> "0", + "f_qt" -> "0", "f_realp" -> "0", - "f_wma" -> "0", - "f_dir" -> "0", - "f_fla" -> "0", - "f_java" -> "1", + "f_wma" -> "0", + "f_dir" -> "0", + "f_fla" -> "0", + "f_java" -> "1", "f_gears" -> "0", - "f_ag" -> "1", - "res" -> "975x571", - "cookie" -> "1", - "url" -> "http://www.psychicbazaar.com/oracles/107-magdalene-oracle.html" - ) ! - "67.71.16.237".some ! "Mozilla/5.0%20(Windows%20NT%206.1;%20Trident/7.0;%20rv:11.0)%20like%20Gecko".some ! "http://www.psychicbazaar.com/oracles/107-magdalene-oracle.html".some | + "f_ag" -> "1", + "res" -> "975x571", + "cookie" -> "1", + "url" -> "http://www.psychicbazaar.com/oracles/107-magdalene-oracle.html" + ) ! + "67.71.16.237".some ! "Mozilla/5.0%20(Windows%20NT%206.1;%20Trident/7.0;%20rv:11.0)%20like%20Gecko".some ! "http://www.psychicbazaar.com/oracles/107-magdalene-oracle.html".some | "CloudFront with x-forwarded-for" !! "2016-07-01 13:17:26 AMS50 480 255.255.255.255 GET d1f6ajd7ltcrsx.cloudfront.net /i 200 http://www.simplybusiness.co.uk/knowledge/articles/2016/06/guide-to-facebook-professional-services-for-small-business/ Mozilla/5.0%20(Windows%20NT%206.1;%20Trident/7.0;%20rv:11.0)%20like%20Gecko e=pv&url=http%253A%252F%252Fwww.simplybusiness.co.uk%252Fknowledge%252Farticles%252F2016%252F06%252Fguide-to-facebook-professional-services-for-small-business%252F&page=Guide%2520to%2520Facebook%2520Professional%2520Services%2520for%2520small%2520business&tv=js-2.4.0&tna=sb-cf-pv&p=web&tz=Europe%252FLondon&lang=en-US&cs=UTF-8&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=0&f_gears=0&f_ag=0&res=1600x900&cd=24&cookie=1&eid=e3793bd1-fcf5-4fbb-bf4c-f0315a5821c3&dtm=1467379046723&vp=1600x799&ds=1583x4043&vid=1&duid=685e511b67c86d5c&fp=2811351631 - Hit LLzvdlIbJ0d6siOm-EY3-2nBYTiM6b5RZLWRyPbyTCE-RIE9bC7_eQ== d1f6ajd7ltcrsx.cloudfront.net http 1627 0.003 67.71.16.237,%20202.134.75.113 - - Hit" ! DateTime.parse("2016-07-01T13:17:26.000+00:00") ! toNameValuePairs( - "e" -> "pv", - "url" -> "http://www.simplybusiness.co.uk/knowledge/articles/2016/06/guide-to-facebook-professional-services-for-small-business/", - "page" -> "Guide to Facebook Professional Services for small business", - "tv" -> "js-2.4.0", - "tna" -> "sb-cf-pv", - "p" -> "web", - "tz" -> "Europe/London", - "lang" -> "en-US", - "cs" -> "UTF-8", - "f_pdf" -> "1", - "f_qt" -> "0", + "e" -> "pv", + "url" -> "http://www.simplybusiness.co.uk/knowledge/articles/2016/06/guide-to-facebook-professional-services-for-small-business/", + "page" -> "Guide to Facebook Professional Services for small business", + "tv" -> "js-2.4.0", + "tna" -> "sb-cf-pv", + "p" -> "web", + "tz" -> "Europe/London", + "lang" -> "en-US", + "cs" -> "UTF-8", + "f_pdf" -> "1", + "f_qt" -> "0", "f_realp" -> "0", - "f_wma" -> "0", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "0", + "f_wma" -> "0", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "0", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "1600x900", - "cd" -> "24", - "cookie" -> "1", - "eid" -> "e3793bd1-fcf5-4fbb-bf4c-f0315a5821c3", - "dtm" -> "1467379046723", - "vp" -> "1600x799", - "ds" -> "1583x4043", - "vid" -> "1", - "duid" -> "685e511b67c86d5c", - "fp" -> "2811351631" - ) ! + "f_ag" -> "0", + "res" -> "1600x900", + "cd" -> "24", + "cookie" -> "1", + "eid" -> "e3793bd1-fcf5-4fbb-bf4c-f0315a5821c3", + "dtm" -> "1467379046723", + "vp" -> "1600x799", + "ds" -> "1583x4043", + "vid" -> "1", + "duid" -> "685e511b67c86d5c", + "fp" -> "2811351631" + ) ! "67.71.16.237".some ! "Mozilla/5.0%20(Windows%20NT%206.1;%20Trident/7.0;%20rv:11.0)%20like%20Gecko".some ! "http://www.simplybusiness.co.uk/knowledge/articles/2016/06/guide-to-facebook-professional-services-for-small-business/".some |> { (_, raw, timestamp, payload, ipAddress, userAgent, refererUri) => @@ -220,12 +228,12 @@ class CloudfrontLoaderSpec extends Specification with DataTables with Validation .toCollectorPayload(raw) val expected = CollectorPayload( - api = Expected.api, + api = Expected.api, querystring = payload, - body = None, + body = None, contentType = None, - source = CollectorSource(Expected.collector, Expected.encoding, None), - context = CollectorContext(timestamp.some, ipAddress, userAgent, refererUri, Nil, None) + source = CollectorSource(Expected.collector, Expected.encoding, None), + context = CollectorContext(timestamp.some, ipAddress, userAgent, refererUri, Nil, None) ) canonicalEvent must beSuccessful(expected.some) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala index e9a10380a..9a9cb1c86 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala @@ -23,23 +23,26 @@ class IpAddressExtractorSpec extends Specification with DataTables with Validati "extractIpAddress" should { "correctly extract an X-FORWARDED-FOR header" in { - "SPEC NAME" || "HEADERS" | "EXP. RESULT" | - "No headers" !! Nil ! Default | - "No X-FORWARDED-FOR header" !! List("Accept-Charset: utf-8", "Connection: keep-alive") ! Default | - "Unparseable X-FORWARDED-FOR header" !! List("X-Forwarded-For: localhost") ! Default | - "Good X-FORWARDED-FOR header" !! List("Accept-Charset: utf-8", - "X-Forwarded-For: 129.78.138.66, 129.78.64.103", - "Connection: keep-alive") ! "129.78.138.66" | - "Good incorrectly capitalized X-FORWARDED-FOR header" !! List("Accept-Charset: utf-8", - "x-FoRwaRdeD-FOr: 129.78.138.66, 129.78.64.103", - "Connection: keep-alive") ! "129.78.138.66" | - "IPv6 address in X-FORWARDED-FOR header" !! List("X-Forwarded-For: 2001:0db8:85a3:0000:0000:8a2e:0370:7334") ! "2001:0db8:85a3:0000:0000:8a2e:0370:7334" | - "IPv6 quoted address in X-FORWARDED-FOR header" !! List( + "SPEC NAME" || "HEADERS" | "EXP. RESULT" | + "No headers" !! Nil ! Default | + "No X-FORWARDED-FOR header" !! List("Accept-Charset: utf-8", "Connection: keep-alive") ! Default | + "Unparseable X-FORWARDED-FOR header" !! List("X-Forwarded-For: localhost") ! Default | + "Good X-FORWARDED-FOR header" !! List( + "Accept-Charset: utf-8", + "X-Forwarded-For: 129.78.138.66, 129.78.64.103", + "Connection: keep-alive") ! "129.78.138.66" | + "Good incorrectly capitalized X-FORWARDED-FOR header" !! List( + "Accept-Charset: utf-8", + "x-FoRwaRdeD-FOr: 129.78.138.66, 129.78.64.103", + "Connection: keep-alive") ! "129.78.138.66" | + "IPv6 address in X-FORWARDED-FOR header" !! List( + "X-Forwarded-For: 2001:0db8:85a3:0000:0000:8a2e:0370:7334") ! "2001:0db8:85a3:0000:0000:8a2e:0370:7334" | + "IPv6 quoted address in X-FORWARDED-FOR header" !! List( "X-Forwarded-For: \"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]\"") ! "2001:0db8:85a3:0000:0000:8a2e:0370:7334" | - "IPv4 address in Forwarded header" !! List("Forwarded: for=129.78.138.66, 129.78.64.103") ! "129.78.138.66" | - "IPv6 incorrectly quoted address in Forwarded header" !! List( + "IPv4 address in Forwarded header" !! List("Forwarded: for=129.78.138.66, 129.78.64.103") ! "129.78.138.66" | + "IPv6 incorrectly quoted address in Forwarded header" !! List( "Forwarded: for=2001:0db8:85a3:0000:0000:8a2e:0370:7334, for=129.78.138.56") ! "2001:0db8:85a3:0000:0000:8a2e:0370:7334" | - "IPv6 quoted correctly in Forwarded header" !! List( + "IPv6 quoted correctly in Forwarded header" !! List( "Forwarded: for=\"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]\", \"129.78.128.66\"") ! "2001:0db8:85a3:0000:0000:8a2e:0370:7334" |> { (_, headers, expected) => { @@ -67,10 +70,10 @@ class IpAddressExtractorSpec extends Specification with DataTables with Validati "correctly extract an X-FORWARDED-FOR Cloudfront field" in { - "SPEC NAME" || "Field" | "EXP. RESULT" | - "No X-FORWARDED-FOR field" !! "-" ! Default | - "Incorrect X-FORWARDED-FOR field" !! "incorrect" ! Default | - "One IP in X-FORWARDED-FOR field" !! "129.78.138.66" ! "129.78.138.66" | + "SPEC NAME" || "Field" | "EXP. RESULT" | + "No X-FORWARDED-FOR field" !! "-" ! Default | + "Incorrect X-FORWARDED-FOR field" !! "incorrect" ! Default | + "One IP in X-FORWARDED-FOR field" !! "129.78.138.66" ! "129.78.138.66" | "Two IPs in X-FORWARDED-FOR field" !! "129.78.138.66,%20129.78.64.103" ! "129.78.138.66" |> { (_, xForwardedFor, expected) => diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala index 7c9d2edd5..228a75965 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala @@ -24,7 +24,6 @@ import Scalaz._ import SpecHelpers._ object LoaderSpec { - val loader = new Loader[String] { // Make our trait whole def toCollectorPayload(line: String): ValidatedMaybeCollectorPayload = "FAIL".failNel @@ -32,11 +31,9 @@ object LoaderSpec { } class LoaderSpec extends Specification with DataTables with ValidationMatchers { - import LoaderSpec._ "getLoader" should { - "return the CloudfrontLoader" in { Loader.getLoader("cloudfront") must beSuccessful(CloudfrontLoader) } @@ -62,16 +59,17 @@ class LoaderSpec extends Specification with DataTables with ValidationMatchers { // TODO: add more tests "return a Success-boxed NonEmptyList of NameValuePairs for a valid or empty querystring" in { - "SPEC NAME" || "QUERYSTRING" | "EXP. NEL" | - "Simple querystring #1" !! "e=pv&dtm=1376487150616&tid=483686".some ! toNameValuePairs("e" -> "pv", - "dtm" -> "1376487150616", - "tid" -> "483686") | + "SPEC NAME" || "QUERYSTRING" | "EXP. NEL" | + "Simple querystring #1" !! "e=pv&dtm=1376487150616&tid=483686".some ! toNameValuePairs( + "e" -> "pv", + "dtm" -> "1376487150616", + "tid" -> "483686") | "Simple querystring #2" !! "page=Celestial%2520Tarot%2520-%2520Psychic%2520Bazaar&vp=1097x482&ds=1097x1973".some ! toNameValuePairs( "page" -> "Celestial%20Tarot%20-%20Psychic%20Bazaar", - "vp" -> "1097x482", - "ds" -> "1097x1973") | + "vp" -> "1097x482", + "ds" -> "1097x1973") | "Superfluous ? ends up in first param's name" !! "?e=pv&dtm=1376487150616&tid=483686".some ! toNameValuePairs( - "?e" -> "pv", + "?e" -> "pv", "dtm" -> "1376487150616", "tid" -> "483686") | "Empty querystring" !! None ! toNameValuePairs() |> { (_, qs, expected) => diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/NdjsonLoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/NdjsonLoaderSpec.scala index cba1e6c63..8d7b2ca4f 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/NdjsonLoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/NdjsonLoaderSpec.scala @@ -19,14 +19,13 @@ import scalaz._ class NdjsonLoaderSpec extends Specification with ValidationMatchers { "toCollectorPayload" should { - "return failure on unparsable json" in { val invalid = NdjsonLoader("com.abc/v1").toCollectorPayload("""{ ... """) invalid must beFailing } "return success on parsable json" in { - val valid = NdjsonLoader("com.abc/v1").toCollectorPayload("""{ "key": "value" } """") + val valid = NdjsonLoader("com.abc/v1").toCollectorPayload("""{ "key": "value" }""") valid must beSuccessful } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala index 7046eff06..91e1a60e3 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala @@ -23,7 +23,11 @@ import Scalaz._ import SpecHelpers._ -class ThriftLoaderSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class ThriftLoaderSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the ThriftLoader functionality toCollectorPayload should return a CollectorPayload for a valid Thrift CollectorPayload (even if parameterless) $e1 @@ -31,15 +35,17 @@ class ThriftLoaderSpec extends Specification with DataTables with ValidationMatc """ object Expected { - val encoding = "UTF-8" + val encoding = "UTF-8" val collector = "ssc-0.0.1-Stdout" // Note we have since fixed -stdout to be lowercase - val api = CollectorApi("com.snowplowanalytics.snowplow", "tp1") + val api = CollectorApi("com.snowplowanalytics.snowplow", "tp1") } def e1 = - "SPEC NAME" || "RAW" | "EXP. TIMESTAMP" | "EXP. PAYLOAD" | "EXP. HOSTNAME" | "EXP. IP ADDRESS" | "EXP. USER AGENT" | "EXP. REFERER URI" | "EXP. HEADERS" | "EXP. USER ID" | - "Fake params" !! "CgABAAABQ5iGqAYLABQAAAAQc3NjLTAuMC4xLVN0ZG91dAsAHgAAAAVVVEYtOAsAKAAAAAkxMjcuMC4wLjEMACkIAAEAAAABCAACAAAAAQsAAwAAABh0ZXN0UGFyYW09MyZ0ZXN0UGFyYW0yPTQACwAtAAAACTEyNy4wLjAuMQsAMgAAAGhNb3ppbGxhLzUuMCAoWDExOyBMaW51eCB4ODZfNjQpIEFwcGxlV2ViS2l0LzUzNy4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZS8zMS4wLjE2NTAuNjMgU2FmYXJpLzUzNy4zNg8ARgsAAAAIAAAAL0Nvb2tpZTogc3A9YzVmM2EwOWYtNzVmOC00MzA5LWJlYzUtZmVhNTYwZjc4NDU1AAAAGkFjY2VwdC1MYW5ndWFnZTogZW4tVVMsIGVuAAAAJEFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZSwgc2RjaAAAAHRVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoWDExOyBMaW51eCB4ODZfNjQpIEFwcGxlV2ViS2l0LzUzNy4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZS8zMS4wLjE2NTAuNjMgU2FmYXJpLzUzNy4zNgAAAFZBY2NlcHQ6IHRleHQvaHRtbCwgYXBwbGljYXRpb24veGh0bWwreG1sLCBhcHBsaWNhdGlvbi94bWw7cT0wLjksIGltYWdlL3dlYnAsICovKjtxPTAuOAAAABhDYWNoZS1Db250cm9sOiBtYXgtYWdlPTAAAAAWQ29ubmVjdGlvbjoga2VlcC1hbGl2ZQAAABRIb3N0OiAxMjcuMC4wLjE6ODA4MAsAUAAAACRjNWYzYTA5Zi03NWY4LTQzMDktYmVjNS1mZWE1NjBmNzg0NTUA" ! - DateTime.parse("2014-01-16T00:49:58.278+00:00") ! toNameValuePairs("testParam" -> "3", "testParam2" -> "4") ! "127.0.0.1".some ! "127.0.0.1".some ! "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36".some ! None ! List( + "SPEC NAME" || "RAW" | "EXP. TIMESTAMP" | "EXP. PAYLOAD" | "EXP. HOSTNAME" | "EXP. IP ADDRESS" | "EXP. USER AGENT" | "EXP. REFERER URI" | "EXP. HEADERS" | "EXP. USER ID" | + "Fake params" !! "CgABAAABQ5iGqAYLABQAAAAQc3NjLTAuMC4xLVN0ZG91dAsAHgAAAAVVVEYtOAsAKAAAAAkxMjcuMC4wLjEMACkIAAEAAAABCAACAAAAAQsAAwAAABh0ZXN0UGFyYW09MyZ0ZXN0UGFyYW0yPTQACwAtAAAACTEyNy4wLjAuMQsAMgAAAGhNb3ppbGxhLzUuMCAoWDExOyBMaW51eCB4ODZfNjQpIEFwcGxlV2ViS2l0LzUzNy4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZS8zMS4wLjE2NTAuNjMgU2FmYXJpLzUzNy4zNg8ARgsAAAAIAAAAL0Nvb2tpZTogc3A9YzVmM2EwOWYtNzVmOC00MzA5LWJlYzUtZmVhNTYwZjc4NDU1AAAAGkFjY2VwdC1MYW5ndWFnZTogZW4tVVMsIGVuAAAAJEFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZSwgc2RjaAAAAHRVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoWDExOyBMaW51eCB4ODZfNjQpIEFwcGxlV2ViS2l0LzUzNy4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZS8zMS4wLjE2NTAuNjMgU2FmYXJpLzUzNy4zNgAAAFZBY2NlcHQ6IHRleHQvaHRtbCwgYXBwbGljYXRpb24veGh0bWwreG1sLCBhcHBsaWNhdGlvbi94bWw7cT0wLjksIGltYWdlL3dlYnAsICovKjtxPTAuOAAAABhDYWNoZS1Db250cm9sOiBtYXgtYWdlPTAAAAAWQ29ubmVjdGlvbjoga2VlcC1hbGl2ZQAAABRIb3N0OiAxMjcuMC4wLjE6ODA4MAsAUAAAACRjNWYzYTA5Zi03NWY4LTQzMDktYmVjNS1mZWE1NjBmNzg0NTUA" ! + DateTime.parse("2014-01-16T00:49:58.278+00:00") ! toNameValuePairs( + "testParam" -> "3", + "testParam2" -> "4") ! "127.0.0.1".some ! "127.0.0.1".some ! "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36".some ! None ! List( "Cookie: sp=c5f3a09f-75f8-4309-bec5-fea560f78455", "Accept-Language: en-US, en", "Accept-Encoding: gzip, deflate, sdch", @@ -48,43 +54,43 @@ class ThriftLoaderSpec extends Specification with DataTables with ValidationMatc "Cache-Control: max-age=0", "Connection: keep-alive", "Host: 127.0.0.1:8080" - ) ! "c5f3a09f-75f8-4309-bec5-fea560f78455".some | + ) ! "c5f3a09f-75f8-4309-bec5-fea560f78455".some | "Page ping" !! "CgABAAABQ9pNXggLABQAAAAQc3NjLTAuMC4xLVN0ZG91dAsAHgAAAAVVVEYtOAsAKAAAAAgxMC4wLjIuMgwAKQgAAQAAAAEIAAIAAAABCwADAAACZmU9cHAmcGFnZT1Bc3luY2hyb25vdXMrd2Vic2l0ZS93ZWJhcHArZXhhbXBsZXMrZm9yK3Nub3dwbG93LmpzJnBwX21peD0wJnBwX21heD0wJnBwX21peT0wJnBwX21heT0wJmNvPSU3QiUyMnBhZ2UlMjI6JTdCJTIycGFnZV90eXBlJTIyOiUyMnRlc3QlMjIsJTIybGFzdF91cGRhdGVkJHRtcyUyMjoxMzkzMzcyODAwMDAwJTdELCUyMnVzZXIlMjI6JTdCJTIydXNlcl90eXBlJTIyOiUyMnRlc3RlciUyMiU3RCU3RCZkdG09MTM5MDkzNjkzODg1NSZ0aWQ9Nzk3NzQzJnZwPTI1NjB4OTYxJmRzPTI1NjB4OTYxJnZpZD03JmR1aWQ9M2MxNzU3NTQ0ZTM5YmNhNCZwPW1vYiZ0dj1qcy0wLjEzLjEmZnA9MjY5NTkzMDgwMyZhaWQ9Q0ZlMjNhJmxhbmc9ZW4tVVMmY3M9VVRGLTgmdHo9RXVyb3BlL0xvbmRvbiZ1aWQ9YWxleCsxMjMmZl9wZGY9MCZmX3F0PTEmZl9yZWFscD0wJmZfd21hPTAmZl9kaXI9MCZmX2ZsYT0xJmZfamF2YT0wJmZfZ2VhcnM9MCZmX2FnPTAmcmVzPTI1NjB4MTQ0MCZjZD0yNCZjb29raWU9MSZ1cmw9ZmlsZTovL2ZpbGU6Ly8vVXNlcnMvYWxleC9EZXZlbG9wbWVudC9kZXYtZW52aXJvbm1lbnQvZGVtby8xLXRyYWNrZXIvZXZlbnRzLmh0bWwvb3ZlcnJpZGRlbi11cmwvAAsALQAAAAlsb2NhbGhvc3QLADIAAABRTW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTAuOTsgcnY6MjYuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8yNi4wDwBGCwAAAAcAAAAWQ29ubmVjdGlvbjoga2VlcC1hbGl2ZQAAAnBDb29raWU6IF9fdXRtYT0xMTE4NzIyODEuODc4MDg0NDg3LjEzOTAyMzcxMDcuMTM5MDg0ODQ4Ny4xMzkwOTMxNTIxLjY7IF9fdXRtej0xMTE4NzIyODEuMTM5MDIzNzEwNy4xLjEudXRtY3NyPShkaXJlY3QpfHV0bWNjbj0oZGlyZWN0KXx1dG1jbWQ9KG5vbmUpOyBfc3BfaWQuMWZmZj1iODlhNmZhNjMxZWVmYWMyLjEzOTAyMzcxMDcuNi4xMzkwOTMxNTQ1LjEzOTA4NDg2NDE7IGhibGlkPUNQamp1aHZGMDV6a3RQN0o3TTVWbzNOSUdQTEp5MVNGOyBvbGZzaz1vbGZzazU2MjkyMzYzNTYxNzU1NDsgX191dG1jPTExMTg3MjI4MTsgd2NzaWQ9dU1sb2cxUUpWRDdqdWhGWjdNNVZvQkN5UFB5aUJ5U1M7IF9va2x2PTEzOTA5MzE1ODU0NDUlMkN1TWxvZzFRSlZEN2p1aEZaN001Vm9CQ3lQUHlpQnlTUzsgX29rPTk3NTItNTAzLTEwLTUyMjc7IF9va2JrPWNkNCUzRHRydWUlMkN2aTUlM0QwJTJDdmk0JTNEMTM5MDkzMTUyMTEyMyUyQ3ZpMyUzRGFjdGl2ZSUyQ3ZpMiUzRGZhbHNlJTJDdmkxJTNEZmFsc2UlMkNjZDglM0RjaGF0JTJDY2Q2JTNEMCUyQ2NkNSUzRGF3YXklMkNjZDMlM0RmYWxzZSUyQ2NkMiUzRDAlMkNjZDElM0QwJTJDOyBzcD03NWExMzU4My01Yzk5LTQwZTMtODFmYy01NDEwODRkZmM3ODQAAAAeQWNjZXB0LUVuY29kaW5nOiBnemlwLCBkZWZsYXRlAAAAGkFjY2VwdC1MYW5ndWFnZTogZW4tVVMsIGVuAAAAK0FjY2VwdDogaW1hZ2UvcG5nLCBpbWFnZS8qO3E9MC44LCAqLyo7cT0wLjUAAABdVXNlci1BZ2VudDogTW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTAuOTsgcnY6MjYuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8yNi4wAAAAFEhvc3Q6IGxvY2FsaG9zdDo0MDAxCwBQAAAAJDc1YTEzNTgzLTVjOTktNDBlMy04MWZjLTU0MTA4NGRmYzc4NAA=" ! DateTime.parse("2014-01-28T19:22:20.040+00:00") ! toNameValuePairs( - "e" -> "pp", - "page" -> "Asynchronous website/webapp examples for snowplow.js", - "pp_mix" -> "0", - "pp_max" -> "0", - "pp_miy" -> "0", - "pp_may" -> "0", - "co" -> """{"page":{"page_type":"test","last_updated$tms":1393372800000},"user":{"user_type":"tester"}}""", - "dtm" -> "1390936938855", - "tid" -> "797743", - "vp" -> "2560x961", - "ds" -> "2560x961", - "vid" -> "7", - "duid" -> "3c1757544e39bca4", - "p" -> "mob", - "tv" -> "js-0.13.1", - "fp" -> "2695930803", - "aid" -> "CFe23a", - "lang" -> "en-US", - "cs" -> "UTF-8", - "tz" -> "Europe/London", - "uid" -> "alex 123", - "f_pdf" -> "0", - "f_qt" -> "1", + "e" -> "pp", + "page" -> "Asynchronous website/webapp examples for snowplow.js", + "pp_mix" -> "0", + "pp_max" -> "0", + "pp_miy" -> "0", + "pp_may" -> "0", + "co" -> """{"page":{"page_type":"test","last_updated$tms":1393372800000},"user":{"user_type":"tester"}}""", + "dtm" -> "1390936938855", + "tid" -> "797743", + "vp" -> "2560x961", + "ds" -> "2560x961", + "vid" -> "7", + "duid" -> "3c1757544e39bca4", + "p" -> "mob", + "tv" -> "js-0.13.1", + "fp" -> "2695930803", + "aid" -> "CFe23a", + "lang" -> "en-US", + "cs" -> "UTF-8", + "tz" -> "Europe/London", + "uid" -> "alex 123", + "f_pdf" -> "0", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "0", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "0", + "f_wma" -> "0", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "0", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "2560x1440", - "cd" -> "24", - "cookie" -> "1", - "url" -> "file://file:///Users/alex/Development/dev-environment/demo/1-tracker/events.html/overridden-url/" + "f_ag" -> "0", + "res" -> "2560x1440", + "cd" -> "24", + "cookie" -> "1", + "url" -> "file://file:///Users/alex/Development/dev-environment/demo/1-tracker/events.html/overridden-url/" ) ! "localhost".some ! "10.0.2.2".some ! "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:26.0) Gecko/20100101 Firefox/26.0".some ! None ! List( "Connection: keep-alive", "Cookie: __utma=111872281.878084487.1390237107.1390848487.1390931521.6; __utmz=111872281.1390237107.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _sp_id.1fff=b89a6fa631eefac2.1390237107.6.1390931545.1390848641; hblid=CPjjuhvF05zktP7J7M5Vo3NIGPLJy1SF; olfsk=olfsk562923635617554; __utmc=111872281; wcsid=uMlog1QJVD7juhFZ7M5VoBCyPPyiBySS; _oklv=1390931585445%2CuMlog1QJVD7juhFZ7M5VoBCyPPyiBySS; _ok=9752-503-10-5227; _okbk=cd4%3Dtrue%2Cvi5%3D0%2Cvi4%3D1390931521123%2Cvi3%3Dactive%2Cvi2%3Dfalse%2Cvi1%3Dfalse%2Ccd8%3Dchat%2Ccd6%3D0%2Ccd5%3Daway%2Ccd3%3Dfalse%2Ccd2%3D0%2Ccd1%3D0%2C; sp=75a13583-5c99-40e3-81fc-541084dfc784", @@ -93,39 +99,39 @@ class ThriftLoaderSpec extends Specification with DataTables with ValidationMatc "Accept: image/png, image/*;q=0.8, */*;q=0.5", "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:26.0) Gecko/20100101 Firefox/26.0", "Host: localhost:4001" - ) ! "75a13583-5c99-40e3-81fc-541084dfc784".some | + ) ! "75a13583-5c99-40e3-81fc-541084dfc784".some | "Unstructured event" !! "CgABAAABQ9qNGa4LABQAAAAQc3NjLTAuMC4xLVN0ZG91dAsAHgAAAAVVVEYtOAsAKAAAAAgxMC4wLjIuMgwAKQgAAQAAAAEIAAIAAAABCwADAAACeWU9dWUmdWVfbmE9Vmlld2VkK1Byb2R1Y3QmdWVfcHI9JTdCJTIycHJvZHVjdF9pZCUyMjolMjJBU08wMTA0MyUyMiwlMjJjYXRlZ29yeSUyMjolMjJEcmVzc2VzJTIyLCUyMmJyYW5kJTIyOiUyMkFDTUUlMjIsJTIycmV0dXJuaW5nJTIyOnRydWUsJTIycHJpY2UlMjI6NDkuOTUsJTIyc2l6ZXMlMjI6JTVCJTIyeHMlMjIsJTIycyUyMiwlMjJsJTIyLCUyMnhsJTIyLCUyMnh4bCUyMiU1RCwlMjJhdmFpbGFibGVfc2luY2UkZHQlMjI6MTU4MDElN0QmZHRtPTEzOTA5NDExMTUyNjMmdGlkPTY0NzYxNSZ2cD0yNTYweDk2MSZkcz0yNTYweDk2MSZ2aWQ9OCZkdWlkPTNjMTc1NzU0NGUzOWJjYTQmcD1tb2ImdHY9anMtMC4xMy4xJmZwPTI2OTU5MzA4MDMmYWlkPUNGZTIzYSZsYW5nPWVuLVVTJmNzPVVURi04JnR6PUV1cm9wZS9Mb25kb24mdWlkPWFsZXgrMTIzJmZfcGRmPTAmZl9xdD0xJmZfcmVhbHA9MCZmX3dtYT0wJmZfZGlyPTAmZl9mbGE9MSZmX2phdmE9MCZmX2dlYXJzPTAmZl9hZz0wJnJlcz0yNTYweDE0NDAmY2Q9MjQmY29va2llPTEmdXJsPWZpbGU6Ly9maWxlOi8vL1VzZXJzL2FsZXgvRGV2ZWxvcG1lbnQvZGV2LWVudmlyb25tZW50L2RlbW8vMS10cmFja2VyL2V2ZW50cy5odG1sL292ZXJyaWRkZW4tdXJsLwALAC0AAAAJbG9jYWxob3N0CwAyAAAAUU1vemlsbGEvNS4wIChNYWNpbnRvc2g7IEludGVsIE1hYyBPUyBYIDEwLjk7IHJ2OjI2LjApIEdlY2tvLzIwMTAwMTAxIEZpcmVmb3gvMjYuMA8ARgsAAAAHAAAAFkNvbm5lY3Rpb246IGtlZXAtYWxpdmUAAAJwQ29va2llOiBfX3V0bWE9MTExODcyMjgxLjg3ODA4NDQ4Ny4xMzkwMjM3MTA3LjEzOTA4NDg0ODcuMTM5MDkzMTUyMS42OyBfX3V0bXo9MTExODcyMjgxLjEzOTAyMzcxMDcuMS4xLnV0bWNzcj0oZGlyZWN0KXx1dG1jY249KGRpcmVjdCl8dXRtY21kPShub25lKTsgX3NwX2lkLjFmZmY9Yjg5YTZmYTYzMWVlZmFjMi4xMzkwMjM3MTA3LjYuMTM5MDkzMTU0NS4xMzkwODQ4NjQxOyBoYmxpZD1DUGpqdWh2RjA1emt0UDdKN001Vm8zTklHUExKeTFTRjsgb2xmc2s9b2xmc2s1NjI5MjM2MzU2MTc1NTQ7IF9fdXRtYz0xMTE4NzIyODE7IHdjc2lkPXVNbG9nMVFKVkQ3anVoRlo3TTVWb0JDeVBQeWlCeVNTOyBfb2tsdj0xMzkwOTMxNTg1NDQ1JTJDdU1sb2cxUUpWRDdqdWhGWjdNNVZvQkN5UFB5aUJ5U1M7IF9vaz05NzUyLTUwMy0xMC01MjI3OyBfb2tiaz1jZDQlM0R0cnVlJTJDdmk1JTNEMCUyQ3ZpNCUzRDEzOTA5MzE1MjExMjMlMkN2aTMlM0RhY3RpdmUlMkN2aTIlM0RmYWxzZSUyQ3ZpMSUzRGZhbHNlJTJDY2Q4JTNEY2hhdCUyQ2NkNiUzRDAlMkNjZDUlM0Rhd2F5JTJDY2QzJTNEZmFsc2UlMkNjZDIlM0QwJTJDY2QxJTNEMCUyQzsgc3A9NzVhMTM1ODMtNWM5OS00MGUzLTgxZmMtNTQxMDg0ZGZjNzg0AAAAHkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZQAAABpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTLCBlbgAAACtBY2NlcHQ6IGltYWdlL3BuZywgaW1hZ2UvKjtxPTAuOCwgKi8qO3E9MC41AAAAXVVzZXItQWdlbnQ6IE1vemlsbGEvNS4wIChNYWNpbnRvc2g7IEludGVsIE1hYyBPUyBYIDEwLjk7IHJ2OjI2LjApIEdlY2tvLzIwMTAwMTAxIEZpcmVmb3gvMjYuMAAAABRIb3N0OiBsb2NhbGhvc3Q6NDAwMQsAUAAAACQ3NWExMzU4My01Yzk5LTQwZTMtODFmYy01NDEwODRkZmM3ODQA" ! DateTime.parse("2014-01-28T20:31:56.846+00:00") ! toNameValuePairs( - "e" -> "ue", - "ue_na" -> "Viewed Product", - "ue_pr" -> """{"product_id":"ASO01043","category":"Dresses","brand":"ACME","returning":true,"price":49.95,"sizes":["xs","s","l","xl","xxl"],"available_since$dt":15801}""", - "dtm" -> "1390941115263", - "tid" -> "647615", - "vp" -> "2560x961", - "ds" -> "2560x961", - "vid" -> "8", - "duid" -> "3c1757544e39bca4", - "p" -> "mob", - "tv" -> "js-0.13.1", - "fp" -> "2695930803", - "aid" -> "CFe23a", - "lang" -> "en-US", - "cs" -> "UTF-8", - "tz" -> "Europe/London", - "uid" -> "alex 123", - "f_pdf" -> "0", - "f_qt" -> "1", + "e" -> "ue", + "ue_na" -> "Viewed Product", + "ue_pr" -> """{"product_id":"ASO01043","category":"Dresses","brand":"ACME","returning":true,"price":49.95,"sizes":["xs","s","l","xl","xxl"],"available_since$dt":15801}""", + "dtm" -> "1390941115263", + "tid" -> "647615", + "vp" -> "2560x961", + "ds" -> "2560x961", + "vid" -> "8", + "duid" -> "3c1757544e39bca4", + "p" -> "mob", + "tv" -> "js-0.13.1", + "fp" -> "2695930803", + "aid" -> "CFe23a", + "lang" -> "en-US", + "cs" -> "UTF-8", + "tz" -> "Europe/London", + "uid" -> "alex 123", + "f_pdf" -> "0", + "f_qt" -> "1", "f_realp" -> "0", - "f_wma" -> "0", - "f_dir" -> "0", - "f_fla" -> "1", - "f_java" -> "0", + "f_wma" -> "0", + "f_dir" -> "0", + "f_fla" -> "1", + "f_java" -> "0", "f_gears" -> "0", - "f_ag" -> "0", - "res" -> "2560x1440", - "cd" -> "24", - "cookie" -> "1", - "url" -> "file://file:///Users/alex/Development/dev-environment/demo/1-tracker/events.html/overridden-url/" + "f_ag" -> "0", + "res" -> "2560x1440", + "cd" -> "24", + "cookie" -> "1", + "url" -> "file://file:///Users/alex/Development/dev-environment/demo/1-tracker/events.html/overridden-url/" ) ! "localhost".some ! "10.0.2.2".some ! "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:26.0) Gecko/20100101 Firefox/26.0".some ! None ! List( "Connection: keep-alive", "Cookie: __utma=111872281.878084487.1390237107.1390848487.1390931521.6; __utmz=111872281.1390237107.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _sp_id.1fff=b89a6fa631eefac2.1390237107.6.1390931545.1390848641; hblid=CPjjuhvF05zktP7J7M5Vo3NIGPLJy1SF; olfsk=olfsk562923635617554; __utmc=111872281; wcsid=uMlog1QJVD7juhFZ7M5VoBCyPPyiBySS; _oklv=1390931585445%2CuMlog1QJVD7juhFZ7M5VoBCyPPyiBySS; _ok=9752-503-10-5227; _okbk=cd4%3Dtrue%2Cvi5%3D0%2Cvi4%3D1390931521123%2Cvi3%3Dactive%2Cvi2%3Dfalse%2Cvi1%3Dfalse%2Ccd8%3Dchat%2Ccd6%3D0%2Ccd5%3Daway%2Ccd3%3Dfalse%2Ccd2%3D0%2Ccd1%3D0%2C; sp=75a13583-5c99-40e3-81fc-541084dfc784", @@ -134,8 +140,8 @@ class ThriftLoaderSpec extends Specification with DataTables with ValidationMatc "Accept: image/png, image/*;q=0.8, */*;q=0.5", "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:26.0) Gecko/20100101 Firefox/26.0", "Host: localhost:4001" - ) ! "75a13583-5c99-40e3-81fc-541084dfc784".some | - "Parameterless" !! "CgABAAABQ9o8zYULABQAAAAQc3NjLTAuMC4xLVN0ZG91dAsAHgAAAAVVVEYtOAsAKAAAAAgxMC4wLjIuMgwAKQgAAQAAAAEIAAIAAAABAAsALQAAAAlsb2NhbGhvc3QLADIAAABRTW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTAuOTsgcnY6MjYuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8yNi4wDwBGCwAAAAgAAAAYQ2FjaGUtQ29udHJvbDogbWF4LWFnZT0wAAAAFkNvbm5lY3Rpb246IGtlZXAtYWxpdmUAAAJwQ29va2llOiBfX3V0bWE9MTExODcyMjgxLjg3ODA4NDQ4Ny4xMzkwMjM3MTA3LjEzOTA4NDg0ODcuMTM5MDkzMTUyMS42OyBfX3V0bXo9MTExODcyMjgxLjEzOTAyMzcxMDcuMS4xLnV0bWNzcj0oZGlyZWN0KXx1dG1jY249KGRpcmVjdCl8dXRtY21kPShub25lKTsgX3NwX2lkLjFmZmY9Yjg5YTZmYTYzMWVlZmFjMi4xMzkwMjM3MTA3LjYuMTM5MDkzMTU0NS4xMzkwODQ4NjQxOyBoYmxpZD1DUGpqdWh2RjA1emt0UDdKN001Vm8zTklHUExKeTFTRjsgb2xmc2s9b2xmc2s1NjI5MjM2MzU2MTc1NTQ7IF9fdXRtYz0xMTE4NzIyODE7IHdjc2lkPXVNbG9nMVFKVkQ3anVoRlo3TTVWb0JDeVBQeWlCeVNTOyBfb2tsdj0xMzkwOTMxNTg1NDQ1JTJDdU1sb2cxUUpWRDdqdWhGWjdNNVZvQkN5UFB5aUJ5U1M7IF9vaz05NzUyLTUwMy0xMC01MjI3OyBfb2tiaz1jZDQlM0R0cnVlJTJDdmk1JTNEMCUyQ3ZpNCUzRDEzOTA5MzE1MjExMjMlMkN2aTMlM0RhY3RpdmUlMkN2aTIlM0RmYWxzZSUyQ3ZpMSUzRGZhbHNlJTJDY2Q4JTNEY2hhdCUyQ2NkNiUzRDAlMkNjZDUlM0Rhd2F5JTJDY2QzJTNEZmFsc2UlMkNjZDIlM0QwJTJDY2QxJTNEMCUyQzsgc3A9NzVhMTM1ODMtNWM5OS00MGUzLTgxZmMtNTQxMDg0ZGZjNzg0AAAAHkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZQAAABpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTLCBlbgAAAEpBY2NlcHQ6IHRleHQvaHRtbCwgYXBwbGljYXRpb24veGh0bWwreG1sLCBhcHBsaWNhdGlvbi94bWw7cT0wLjksICovKjtxPTAuOAAAAF1Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoTWFjaW50b3NoOyBJbnRlbCBNYWMgT1MgWCAxMC45OyBydjoyNi4wKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzI2LjAAAAAUSG9zdDogbG9jYWxob3N0OjQwMDELAFAAAAAkNzVhMTM1ODMtNWM5OS00MGUzLTgxZmMtNTQxMDg0ZGZjNzg0AA==" ! + ) ! "75a13583-5c99-40e3-81fc-541084dfc784".some | + "Parameterless" !! "CgABAAABQ9o8zYULABQAAAAQc3NjLTAuMC4xLVN0ZG91dAsAHgAAAAVVVEYtOAsAKAAAAAgxMC4wLjIuMgwAKQgAAQAAAAEIAAIAAAABAAsALQAAAAlsb2NhbGhvc3QLADIAAABRTW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTAuOTsgcnY6MjYuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8yNi4wDwBGCwAAAAgAAAAYQ2FjaGUtQ29udHJvbDogbWF4LWFnZT0wAAAAFkNvbm5lY3Rpb246IGtlZXAtYWxpdmUAAAJwQ29va2llOiBfX3V0bWE9MTExODcyMjgxLjg3ODA4NDQ4Ny4xMzkwMjM3MTA3LjEzOTA4NDg0ODcuMTM5MDkzMTUyMS42OyBfX3V0bXo9MTExODcyMjgxLjEzOTAyMzcxMDcuMS4xLnV0bWNzcj0oZGlyZWN0KXx1dG1jY249KGRpcmVjdCl8dXRtY21kPShub25lKTsgX3NwX2lkLjFmZmY9Yjg5YTZmYTYzMWVlZmFjMi4xMzkwMjM3MTA3LjYuMTM5MDkzMTU0NS4xMzkwODQ4NjQxOyBoYmxpZD1DUGpqdWh2RjA1emt0UDdKN001Vm8zTklHUExKeTFTRjsgb2xmc2s9b2xmc2s1NjI5MjM2MzU2MTc1NTQ7IF9fdXRtYz0xMTE4NzIyODE7IHdjc2lkPXVNbG9nMVFKVkQ3anVoRlo3TTVWb0JDeVBQeWlCeVNTOyBfb2tsdj0xMzkwOTMxNTg1NDQ1JTJDdU1sb2cxUUpWRDdqdWhGWjdNNVZvQkN5UFB5aUJ5U1M7IF9vaz05NzUyLTUwMy0xMC01MjI3OyBfb2tiaz1jZDQlM0R0cnVlJTJDdmk1JTNEMCUyQ3ZpNCUzRDEzOTA5MzE1MjExMjMlMkN2aTMlM0RhY3RpdmUlMkN2aTIlM0RmYWxzZSUyQ3ZpMSUzRGZhbHNlJTJDY2Q4JTNEY2hhdCUyQ2NkNiUzRDAlMkNjZDUlM0Rhd2F5JTJDY2QzJTNEZmFsc2UlMkNjZDIlM0QwJTJDY2QxJTNEMCUyQzsgc3A9NzVhMTM1ODMtNWM5OS00MGUzLTgxZmMtNTQxMDg0ZGZjNzg0AAAAHkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZQAAABpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTLCBlbgAAAEpBY2NlcHQ6IHRleHQvaHRtbCwgYXBwbGljYXRpb24veGh0bWwreG1sLCBhcHBsaWNhdGlvbi94bWw7cT0wLjksICovKjtxPTAuOAAAAF1Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoTWFjaW50b3NoOyBJbnRlbCBNYWMgT1MgWCAxMC45OyBydjoyNi4wKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzI2LjAAAAAUSG9zdDogbG9jYWxob3N0OjQwMDELAFAAAAAkNzVhMTM1ODMtNWM5OS00MGUzLTgxZmMtNTQxMDg0ZGZjNzg0AA==" ! DateTime.parse("2014-01-28T19:04:14.469+00:00") ! toNameValuePairs() ! "localhost".some ! "10.0.2.2".some ! "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:26.0) Gecko/20100101 Firefox/26.0".some ! None ! List( "Cache-Control: max-age=0", "Connection: keep-alive", @@ -153,12 +159,13 @@ class ThriftLoaderSpec extends Specification with DataTables with ValidationMatc val canonicalEvent = ThriftLoader.toCollectorPayload(Base64.decodeBase64(raw)) val expected = CollectorPayload( - api = Expected.api, + api = Expected.api, querystring = payload, - body = None, + body = None, contentType = None, - source = CollectorSource(Expected.collector, Expected.encoding, hostname), - context = CollectorContext(timestamp.some, ipAddress, userAgent, refererUri, headers, userId) + source = CollectorSource(Expected.collector, Expected.encoding, hostname), + context = + CollectorContext(timestamp.some, ipAddress, userAgent, refererUri, headers, userId) ) canonicalEvent must beSuccessful(expected.some) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/TsvLoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/TsvLoaderSpec.scala index 9e652ab1c..2940a4364 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/TsvLoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/TsvLoaderSpec.scala @@ -27,16 +27,18 @@ class TsvLoaderSpec extends Specification with DataTables with ValidationMatcher def e1 = { val expected = CollectorPayload( - api = CollectorApi("com.amazon.aws.cloudfront", "wd_access_log"), + api = CollectorApi("com.amazon.aws.cloudfront", "wd_access_log"), querystring = Nil, - body = "a\tb".some, + body = "a\tb".some, contentType = None, - source = CollectorSource("tsv", "UTF-8", None), - context = CollectorContext(None, None, None, None, Nil, None) + source = CollectorSource("tsv", "UTF-8", None), + context = CollectorContext(None, None, None, None, Nil, None) ) - TsvLoader("com.amazon.aws.cloudfront/wd_access_log").toCollectorPayload("a\tb") must beSuccessful(expected.some) + TsvLoader("com.amazon.aws.cloudfront/wd_access_log").toCollectorPayload("a\tb") must beSuccessful( + expected.some) } def e2 = - TsvLoader("com.amazon.aws.cloudfront/wd_access_log").toCollectorPayload("#Version: 1.0") must beSuccessful(None) + TsvLoader("com.amazon.aws.cloudfront/wd_access_log").toCollectorPayload("#Version: 1.0") must beSuccessful( + None) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/collectorPayloadSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/collectorPayloadSpecs.scala index e2a707c41..bb37c472d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/collectorPayloadSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/collectorPayloadSpecs.scala @@ -22,13 +22,12 @@ class CollectorApiSpec extends Specification with DataTables with ValidationMatc // (then we can make isIceRequest private again). "isIceRequest" should { "correctly identify valid Snowplow GET requests" in { - - "SPEC NAME" || "PATH" | "EXP. RESULT" | - "Valid #1" !! "/i" ! true | - "Valid #2" !! "/ice.png" ! true | - "Valid #3" !! "/i?foo=1&bar=2" ! true | - "Invalid #1" !! "/blah/i" ! false | - "Invalid #2" !! "i" ! false |> { (_, path, expected) => + "SPEC NAME" || "PATH" | "EXP. RESULT" | + "Valid #1" !! "/i" ! true | + "Valid #2" !! "/ice.png" ! true | + "Valid #3" !! "/i?foo=1&bar=2" ! true | + "Invalid #1" !! "/blah/i" ! false | + "Invalid #2" !! "i" ! false |> { (_, path, expected) => { CollectorApi.isIceRequest(path) must_== expected } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala index 5f22f3192..a007bfa29 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala @@ -12,8 +12,8 @@ */ package com.snowplowanalytics.snowplow.enrich.common.utils -import org.json4s._ -import org.json4s.jackson.parseJson +import io.circe._ +import io.circe.syntax._ import org.specs2.Specification import org.specs2.scalaz.ValidationMatchers @@ -26,52 +26,56 @@ class JsonPathSpec extends Specification with ValidationMatchers { Test primtive JSON type (JString) $e6 Invalid JSONPath (JQ syntax) must fail $e4 Invalid JSONPath must fail $e5 - JNothing must fail $e7 """ - val someJson = parseJson(""" - |{ "store": { - | "book": [ - | { "category": "reference", - | "author": "Nigel Rees", - | "title": "Sayings of the Century", - | "price": 8.95 - | }, - | { "category": "fiction", - | "author": "Evelyn Waugh", - | "title": "Sword of Honour", - | "price": 12.99 - | }, - | { "category": "fiction", - | "author": "Herman Melville", - | "title": "Moby Dick", - | "isbn": "0-553-21311-3", - | "price": 8.99 - | }, - | { "category": "fiction", - | "author": "J. R. R. Tolkien", - | "title": "The Lord of the Rings", - | "isbn": "0-395-19395-8", - | "price": 22.99 - | } - | ], - | "bicycle": { - | "color": "red", - | "price": 19.95 - | }, - | "unicorns": [] - | } - |} - """.stripMargin) + val someJson = Json.obj( + "store" := Json.obj( + "book" := Json.fromValues( + List( + Json.obj( + "category" := Json.fromString("reference"), + "author" := Json.fromString("Nigel Rees"), + "title" := Json.fromString("Savings of the Century"), + "price" := Json.fromDoubleOrNull(8.95) + ), + Json.obj( + "category" := Json.fromString("fiction"), + "author" := Json.fromString("Evelyn Waugh"), + "title" := Json.fromString("Swords of Honour"), + "price" := Json.fromDoubleOrNull(12.99) + ), + Json.obj( + "category" := Json.fromString("fiction"), + "author" := Json.fromString("Herman Melville"), + "title" := Json.fromString("Moby Dick"), + "isbn" := Json.fromString("0-553-21311-3"), + "price" := Json.fromDoubleOrNull(8.99) + ), + Json.obj( + "category" := Json.fromString("fiction"), + "author" := Json.fromString("J. R. R. Tolkien"), + "title" := Json.fromString("The Lord of the Rings"), + "isbn" := Json.fromString("0-395-19395-8"), + "price" := Json.fromDoubleOrNull(22.99) + ) + )), + "bicycles" := Json.obj( + "color" := Json.fromString("red"), + "price" := Json.fromDoubleOrNull(19.95) + ), + "unicors" := Json.fromValues(Nil) + ) + ) def e1 = - JsonPath.query("$.store.book[1].price", someJson) must beSuccessful(List(JDouble(12.99))) + JsonPath.query("$.store.book[1].price", someJson) must + beSuccessful(List(Json.fromDoubleOrNull(12.99))) def e2 = JsonPath.query("$.store.book[5].price", someJson) must beSuccessful(Nil) def e3 = - JsonPath.query("$.store.unicorns", someJson) must beSuccessful(List(JArray(Nil))) + JsonPath.query("$.store.unicorns", someJson) must beSuccessful(Nil) def e4 = JsonPath.query(".notJsonPath", someJson) must beFailing.like { @@ -84,10 +88,5 @@ class JsonPathSpec extends Specification with ValidationMatchers { } def e6 = - JsonPath.query("$.store.book[2]", JString("somestring")) must beSuccessful(List()) - - def e7 = - JsonPath.query("$..", JNothing) must beFailing.like { - case f => f must beEqualTo("JSONPath error: Nothing was given") - } + JsonPath.query("$.store.book[2]", Json.fromString("somestring")) must beSuccessful(List()) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala index 5caaa6b26..208c44e6f 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala @@ -25,12 +25,12 @@ import enrichments.{ClientEnrichments, MiscEnrichments} // Test Bean final class TargetBean { - @BeanProperty var platform: String = _ + @BeanProperty var platform: String = _ @BeanProperty var br_features_pdf: Byte = _ - @BeanProperty var visit_id: Int = _ - @BeanProperty var tracker_v: String = _ - @BeanProperty var width: Int = _ - @BeanProperty var height: Int = _ + @BeanProperty var visit_id: Int = _ + @BeanProperty var tracker_v: String = _ + @BeanProperty var width: Int = _ + @BeanProperty var height: Int = _ override def equals(other: Any): Boolean = other match { case that: TargetBean => { @@ -46,21 +46,19 @@ final class TargetBean { // No canEqual needed as the class is final // Use Reflection - perf hit is okay as this is only in the test suite - override def hashCode: Int = HashCodeBuilder.reflectionHashCode(this, false) + override def hashCode: Int = HashCodeBuilder.reflectionHashCode(this, false) override def toString: String = ToStringBuilder.reflectionToString(this) } -/** - * Tests the MapTransformer. - */ class MapTransformerSpec extends Specification with ValidationMatchers { - val sourceMap = Map("p" -> "web", - "f_pdf" -> "1", - "vid" -> "1", - "tv" -> "no-js-0.1.1", - "res" -> "720x1080", - "missing" -> "Not in the transformation map") + val sourceMap = Map( + "p" -> "web", + "f_pdf" -> "1", + "vid" -> "1", + "tv" -> "no-js-0.1.1", + "res" -> "720x1080", + "missing" -> "Not in the transformation map") val transformMap: TransformMap = Map( ("p", (MiscEnrichments.extractPlatform, "platform")), @@ -72,21 +70,20 @@ class MapTransformerSpec extends Specification with ValidationMatchers { val expected = { val t = new TargetBean() - t.platform = "web" + t.platform = "web" t.br_features_pdf = 1 - t.visit_id = 1 - t.tracker_v = "no-js-0.1.1" - t.width = 720 - t.height = 1080 + t.visit_id = 1 + t.tracker_v = "no-js-0.1.1" + t.width = 720 + t.height = 1080 t } "Applying a TransformMap to an existing POJO" should { "successfully set each of the target fields" in { - val target = { val t = new TargetBean() - t.platform = "old" + t.platform = "old" t.tracker_v = "old" t } @@ -99,7 +96,6 @@ class MapTransformerSpec extends Specification with ValidationMatchers { "Executing TransformMap's generate() factory" should { "successfully instantiate a new POJO" in { - val result = MapTransformer.generate[TargetBean](sourceMap, transformMap) result must beSuccessful(expected) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ScalazJson4sSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ScalazCirceSpec.scala similarity index 75% rename from modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ScalazJson4sSpec.scala rename to modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ScalazCirceSpec.scala index 9b61b1ae2..46f4a9b7b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ScalazJson4sSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ScalazCirceSpec.scala @@ -13,33 +13,28 @@ package com.snowplowanalytics.snowplow.enrich.common package utils -import org.json4s.jackson.JsonMethods.parse -import org.json4s.DefaultFormats +import io.circe.literal._ import org.specs2.mutable.Specification import org.specs2.scalaz.ValidationMatchers -class JsonExtractionSpec extends Specification with ValidationMatchers { - implicit val formats = DefaultFormats - val testJson = parse("""{ +class ScalazCirceSpec extends Specification with ValidationMatchers { + val testJson = json"""{ "outer": "1", "inner": { "value": 2 } - }""") + }""" "Applying extractString" should { "successfully access an outer string field" in { - - val result = ScalazJson4sUtils.extract[String](testJson, "outer") + val result = ScalazCirceUtils.extract[String](testJson, "outer") result must beSuccessful("1") } } "Applying extractInt" should { - "successfully access an inner string field" in { - - val result = ScalazJson4sUtils.extract[Int](testJson, "inner", "value") + val result = ScalazCirceUtils.extract[Int](testJson, "inner", "value") result must beSuccessful(2) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/TestResourcesRepositoryRef.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/TestResourcesRepositoryRef.scala index 5b24c6757..49150751c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/TestResourcesRepositoryRef.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/TestResourcesRepositoryRef.scala @@ -25,47 +25,39 @@ import com.snowplowanalytics.iglu.client.utils.{ValidationExceptions => VE} import com.snowplowanalytics.iglu.client.validation.ProcessingMessageMethods._ import scalaz.Scalaz._ -/** - * Iglu repository ref that looks up a schema in test resources. - */ -case class TestResourcesRepositoryRef(override val config: RepositoryRefConfig, path: String) extends RepositoryRef { +/** Iglu repository ref that looks up a schema in test resources. */ +case class TestResourcesRepositoryRef( + override val config: RepositoryRefConfig, + path: String +) extends RepositoryRef { - /** - * Prioritize searching this class of repository to ensure that tests use it - */ + /** Prioritize searching this class of repository to ensure that tests use it */ override val classPriority: Int = 1 - /** - * Human-readable descriptor for this - * type of repository ref. - */ + /** Human-readable descriptor for this type of repository ref. */ val descriptor = "test" /** - * Retrieves an IgluSchema from the Iglu Repo as - * a JsonNode. - * - * @param schemaKey The SchemaKey uniquely identifies - * the schema in Iglu - * @return a Validation boxing either the Schema's - * JsonNode on Success, or an error String - * on Failure + * Retrieves an IgluSchema from the Iglu Repo as a JsonNode. + * @param schemaKey The SchemaKey uniquely identifies the schema in Iglu + * @return a Validation boxing either the Schema's JsonNode on Success, or an error String + * on Failure */ def lookupSchema(schemaKey: SchemaKey): Validated[Option[JsonNode]] = { - val schemaPath = s"${path}/${schemaKey.toPath}" + val schemaPath = s"$path/${schemaKey.toPath}" try { JsonLoader.fromPath(schemaPath).some.success } catch { case jpe: JsonParseException => // Child of IOException so match first - s"Problem parsing ${schemaPath} as JSON in ${descriptor} Iglu repository ${config.name}: %s" + s"Problem parsing $schemaPath as JSON in $descriptor Iglu repository ${config.name}: %s" .format(VE.stripInstanceEtc(jpe.getMessage)) .failure .toProcessingMessage case ioe: IOException => None.success // Schema not found case NonFatal(e) => - s"Unknown problem reading and parsing ${schemaPath} in ${descriptor} Iglu repository ${config.name}: ${VE - .getThrowableMessage(e)}".failure.toProcessingMessage + (s"Unknown problem reading and parsing $schemaPath in $descriptor Iglu repository " + + s"${config.name}: ${VE.getThrowableMessage(e)}").failure.toProcessingMessage } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ValidateAndReformatJsonSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ValidateAndReformatJsonSpec.scala index 58db4154b..2ae4cc251 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ValidateAndReformatJsonSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/ValidateAndReformatJsonSpec.scala @@ -26,41 +26,35 @@ class ValidateAndReformatJsonSpec extends Specification with DataTables with Val val FieldName = "json" def e1 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Empty JSON" !! "{}" ! "{}" | - "Simple JSON #1" !! """{"key":"value"}""" ! """{"key":"value"}""" | - "Simple JSON #2" !! """[1,2,3]""" ! """[1,2,3]""" | - "Tolerated JSON #1" !! """{"a":9}}}""" ! """{"a":9}""" | - "Tolerated JSON #2" !! """[];[]""" ! """[]""" | - "Tolerated JSON #3" !! """"a":[]""" ! "\"a\"" | - "Reformatted JSON #1" !! """{ "key" : 23 }""" ! """{"key":23}""" | - "Reformatted JSON #2" !! """[1.00, 2.00, 3.00, 4.00]""" ! """[1.0,2.0,3.0,4.0]""" | + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Empty JSON" !! "{}" ! "{}" | + "Simple JSON #1" !! """{"key":"value"}""" ! """{"key":"value"}""" | + "Simple JSON #2" !! """[1,2,3]""" ! """[1,2,3]""" | + "Reformatted JSON #1" !! """{ "key" : 23 }""" ! """{"key":23}""" | + "Reformatted JSON #2" !! """[1.00, 2.00, 3.00, 4.00]""" ! """[1.00,2.00,3.00,4.00]""" | "Reformatted JSON #3" !! """ { "a": 23 - }""" ! """{"a":23}""" |> { (_, str, expected) => + }""" ! """{"a":23}""" |> { (_, str, expected) => JsonUtils.validateAndReformatJson(FieldName, str) must beSuccessful(expected) } - def err1 = - """Field [%s]: invalid JSON [] with parsing error: mapping resulted in null""" - .format(FieldName) - def err2: (String, Char, Integer, Integer) => String = - (str, char, code, pos) => - """Field [%s]: invalid JSON [%s] with parsing error: Unexpected character ('%c' (code %d)): expected a valid value (number, String, array, object, 'true', 'false' or 'null') at [Source: (String)"%2$s"; line: 1, column: %d]""" - .format(FieldName, str, char, code, pos) - def err3: (String, Char, Integer) => String = - (str, char, int) => - """Field [%s]: invalid JSON [%s] with parsing error: Unexpected character ('%c' (code %d)): was expecting double-quote to start field name at [Source: (String)"%2$s"; line: 1, column: 3]""" - .format(FieldName, str, char, int) + def err1 = s"Field [$FieldName]: invalid JSON [] with parsing error: exhausted input" + def err2: (String, String, Int, Int) => String = + (str, got, line, col) => + s"Field [$FieldName]: invalid JSON [$str] with parsing error: expected json value got '$got' (line $line, column $col)" + def err3: (String, String, Int, Int) => String = + (str, got, line, col) => + s"""Field [$FieldName]: invalid JSON [$str] with parsing error: expected " got '$got' (line $line, column $col)""" def e2 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Empty string" !! "" ! err1 | - "Double colons" !! """{"a"::2}""" ! err2("""{"a"::2}""", ':', 58, 7) | - "Random noise" !! "^45fj_" ! err2("^45fj_", '^', 94, 2) | - "Bad key" !! """{9:"a"}""" ! err3("""{9:"a"}""", '9', 57) |> { (_, str, expected) => - JsonUtils.validateAndReformatJson(FieldName, str) must beFailing(expected) + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Empty string" !! "" ! err1 | + "Double colons" !! """{"a"::2}""" ! err2("""{"a"::2}""", ":2}", 1, 6) | + "Random noise" !! "^45fj_" ! err2("^45fj_", "^45fj_", 1, 1) | + "Bad key" !! """{9:"a"}""" ! err3("""{9:"a"}""", """9:"a"}""", 1, 2) |> { + (_, str, expected) => + JsonUtils.validateAndReformatJson(FieldName, str) must beFailing(expected) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala index 165712a6d..78529065c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala @@ -128,27 +128,31 @@ class ExplodeUriSpec extends Specification with DataTables { def is = s2"Exploding URIs into their component pieces with explodeUri should work $e1" def e1 = - "SPEC NAME" || "URI" | "EXP. SCHEME" | "EXP. HOST" | "EXP. PORT" | "EXP. PATH" | "EXP. QUERY" | "EXP. FRAGMENT" | - "With path, qs & #" !! "http://www.psychicbazaar.com/oracles/119-psycards-deck.html?view=print#detail" ! "http" ! "www.psychicbazaar.com" ! 80 ! Some( - "/oracles/119-psycards-deck.html") ! Some("view=print") ! Some("detail") | - "With path & space in qs" !! "http://psy.bz/genre/all/type/all?utm_source=google&utm_medium=cpc&utm_term=buy%2Btarot&utm_campaign=spring_sale" ! "http" ! "psy.bz" ! 80 ! Some( - "/genre/all/type/all") ! Some("utm_source=google&utm_medium=cpc&utm_term=buy%2Btarot&utm_campaign=spring_sale") ! None | - "With path & no www" !! "http://snowplowanalytics.com/analytics/index.html" ! "http" ! "snowplowanalytics.com" ! 80 ! Some( - "/analytics/index.html") ! None ! None | - "Port specified" !! "http://www.nbnz.co.nz:440/login.asp" ! "http" ! "www.nbnz.co.nz" ! 440 ! Some("/login.asp") ! None ! None | - "HTTPS & #" !! "https://www.lancs.ac.uk#footer" ! "https" ! "www.lancs.ac.uk" ! 443 ! None ! None ! Some("footer") | - "www2 & trailing /" !! "https://www2.williamhill.com/" ! "https" ! "www2.williamhill.com" ! 443 ! Some("/") ! None ! None | - "Tab & newline in qs" !! "http://www.ebay.co.uk/sch/i.html?_from=R40&_trksid=m570.l2736&_nkw=%09+Clear+Quartz+Point+Rock+Crystal%0ADowsing+Pendulum" ! "http" ! "www.ebay.co.uk" ! 80 ! Some( - "/sch/i.html") ! Some( + "SPEC NAME" || "URI" | "EXP. SCHEME" | "EXP. HOST" | "EXP. PORT" | "EXP. PATH" | "EXP. QUERY" | "EXP. FRAGMENT" | + "With path, qs & #" !! "http://www.psychicbazaar.com/oracles/119-psycards-deck.html?view=print#detail" ! "http" ! "www.psychicbazaar.com" ! 80 ! Some( + "/oracles/119-psycards-deck.html") ! Some("view=print") ! Some("detail") | + "With path & space in qs" !! "http://psy.bz/genre/all/type/all?utm_source=google&utm_medium=cpc&utm_term=buy%2Btarot&utm_campaign=spring_sale" ! "http" ! "psy.bz" ! 80 ! Some( + "/genre/all/type/all") ! Some( + "utm_source=google&utm_medium=cpc&utm_term=buy%2Btarot&utm_campaign=spring_sale") ! None | + "With path & no www" !! "http://snowplowanalytics.com/analytics/index.html" ! "http" ! "snowplowanalytics.com" ! 80 ! Some( + "/analytics/index.html") ! None ! None | + "Port specified" !! "http://www.nbnz.co.nz:440/login.asp" ! "http" ! "www.nbnz.co.nz" ! 440 ! Some( + "/login.asp") ! None ! None | + "HTTPS & #" !! "https://www.lancs.ac.uk#footer" ! "https" ! "www.lancs.ac.uk" ! 443 ! None ! None ! Some( + "footer") | + "www2 & trailing /" !! "https://www2.williamhill.com/" ! "https" ! "www2.williamhill.com" ! 443 ! Some( + "/") ! None ! None | + "Tab & newline in qs" !! "http://www.ebay.co.uk/sch/i.html?_from=R40&_trksid=m570.l2736&_nkw=%09+Clear+Quartz+Point+Rock+Crystal%0ADowsing+Pendulum" ! "http" ! "www.ebay.co.uk" ! 80 ! Some( + "/sch/i.html") ! Some( "_from=R40&_trksid=m570.l2736&_nkw=%09+Clear+Quartz+Point+Rock+Crystal%0ADowsing+Pendulum") ! None | - "Tab & newline in path" !! "https://snowplowanalytics.com/analytic%0As/index%09nasty.html" ! "https" ! "snowplowanalytics.com" ! 443 ! Some( - "/analytic%0As/index%09nasty.html") ! None ! None | - "Tab & newline in #" !! "http://psy.bz/oracles/psycards.html?view=print#detail%09is%0Acorrupted" ! "http" ! "psy.bz" ! 80 ! Some( - "/oracles/psycards.html") ! Some("view=print") ! Some("detail%09is%0Acorrupted") |> { + "Tab & newline in path" !! "https://snowplowanalytics.com/analytic%0As/index%09nasty.html" ! "https" ! "snowplowanalytics.com" ! 443 ! Some( + "/analytic%0As/index%09nasty.html") ! None ! None | + "Tab & newline in #" !! "http://psy.bz/oracles/psycards.html?view=print#detail%09is%0Acorrupted" ! "http" ! "psy.bz" ! 80 ! Some( + "/oracles/psycards.html") ! Some("view=print") ! Some("detail%09is%0Acorrupted") |> { (_, uri, scheme, host, port, path, query, fragment) => { - val actual = ConversionUtils.explodeUri(new URI(uri)) + val actual = ConversionUtils.explodeUri(new URI(uri)) val expected = ConversionUtils.UriComponents(scheme, host, port, path, query, fragment) actual must_== expected } @@ -163,25 +167,30 @@ class FixTabsNewlinesSpec extends Specification with DataTables { def is = s2"Replacing tabs, newlines and control characters with fixTabsNewlines should work $e1" def e1 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Empty string" !! "" ! None | - "String with true-tab" !! " " ! SafeTab.some | - "String with \\t" !! "\t" ! SafeTab.some | - "String with \\\\t" !! "\\\t" ! "\\%s".format(SafeTab).some | - "String with \\b" !! "\b" ! None | - "String ending in newline" !! "Hello\n" ! "Hello".some | - "String with control char" !! "\u0002" ! None | - "String with space" !! "\u0020" ! " ".some | - "String with black diamond" !! "�" ! "�".some | - "String with everything" !! "Hi \u0002�\u0020\bJo\t\u0002" ! "Hi%s� Jo%s".format(SafeTab, SafeTab).some |> { - (_, str, expected) => - ConversionUtils.fixTabsNewlines(str) must_== expected + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Empty string" !! "" ! None | + "String with true-tab" !! " " ! SafeTab.some | + "String with \\t" !! "\t" ! SafeTab.some | + "String with \\\\t" !! "\\\t" ! "\\%s".format(SafeTab).some | + "String with \\b" !! "\b" ! None | + "String ending in newline" !! "Hello\n" ! "Hello".some | + "String with control char" !! "\u0002" ! None | + "String with space" !! "\u0020" ! " ".some | + "String with black diamond" !! "�" ! "�".some | + "String with everything" !! "Hi \u0002�\u0020\bJo\t\u0002" ! "Hi%s� Jo%s" + .format(SafeTab, SafeTab) + .some |> { (_, str, expected) => + ConversionUtils.fixTabsNewlines(str) must_== expected } } // TODO: note that we have some functionality tweaks planned. // See comments on ConversionUtils.decodeBase64Url for details. -class DecodeBase64UrlSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class DecodeBase64UrlSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the decodeBase64Url function decodeBase64Url should return failure if passed a null $e1 @@ -207,24 +216,28 @@ class DecodeBase64UrlSpec extends Specification with DataTables with ValidationM // 2. Manual tests of the JavaScript Tracker's trackUnstructEvent() // 3. Misc edge cases worth checking def e3 = - "SPEC NAME" || "ENCODED STRING" | "EXPECTED" | - "Lua Tracker String #1" !! "Sm9oblNtaXRo" ! "JohnSmith" | - "Lua Tracker String #2" !! "am9obitzbWl0aA" ! "john+smith" | - "Lua Tracker String #3" !! "Sm9obiBTbWl0aA" ! "John Smith" | - "Lua Tracker JSON #1" !! "eyJhZ2UiOjIzLCJuYW1lIjoiSm9obiJ9" ! """{"age":23,"name":"John"}""" | - "Lua Tracker JSON #2" !! "eyJteVRlbXAiOjIzLjMsIm15VW5pdCI6ImNlbHNpdXMifQ" ! """{"myTemp":23.3,"myUnit":"celsius"}""" | - "Lua Tracker JSON #3" !! "eyJldmVudCI6InBhZ2VfcGluZyIsIm1vYmlsZSI6dHJ1ZSwicHJvcGVydGllcyI6eyJtYXhfeCI6OTYwLCJtYXhfeSI6MTA4MCwibWluX3giOjAsIm1pbl95IjotMTJ9fQ" ! """{"event":"page_ping","mobile":true,"properties":{"max_x":960,"max_y":1080,"min_x":0,"min_y":-12}}""" | - "Lua Tracker JSON #4" !! "eyJldmVudCI6ImJhc2tldF9jaGFuZ2UiLCJwcmljZSI6MjMuMzksInByb2R1Y3RfaWQiOiJQQlowMDAzNDUiLCJxdWFudGl0eSI6LTIsInRzdGFtcCI6MTY3ODAyMzAwMH0" ! """{"event":"basket_change","price":23.39,"product_id":"PBZ000345","quantity":-2,"tstamp":1678023000}""" | - "JS Tracker JSON #1" !! "eyJwcm9kdWN0X2lkIjoiQVNPMDEwNDMiLCJjYXRlZ29yeSI6IkRyZXNzZXMiLCJicmFuZCI6IkFDTUUiLCJyZXR1cm5pbmciOnRydWUsInByaWNlIjo0OS45NSwic2l6ZXMiOlsieHMiLCJzIiwibCIsInhsIiwieHhsIl0sImF2YWlsYWJsZV9zaW5jZSRkdCI6MTU4MDF9" ! """{"product_id":"ASO01043","category":"Dresses","brand":"ACME","returning":true,"price":49.95,"sizes":["xs","s","l","xl","xxl"],"available_since$dt":15801}""" | - "Unescaped characters" !! "äöü - &" ! "" | - "Blank string" !! "" ! "" |> { (_, str, expected) => + "SPEC NAME" || "ENCODED STRING" | "EXPECTED" | + "Lua Tracker String #1" !! "Sm9oblNtaXRo" ! "JohnSmith" | + "Lua Tracker String #2" !! "am9obitzbWl0aA" ! "john+smith" | + "Lua Tracker String #3" !! "Sm9obiBTbWl0aA" ! "John Smith" | + "Lua Tracker JSON #1" !! "eyJhZ2UiOjIzLCJuYW1lIjoiSm9obiJ9" ! """{"age":23,"name":"John"}""" | + "Lua Tracker JSON #2" !! "eyJteVRlbXAiOjIzLjMsIm15VW5pdCI6ImNlbHNpdXMifQ" ! """{"myTemp":23.3,"myUnit":"celsius"}""" | + "Lua Tracker JSON #3" !! "eyJldmVudCI6InBhZ2VfcGluZyIsIm1vYmlsZSI6dHJ1ZSwicHJvcGVydGllcyI6eyJtYXhfeCI6OTYwLCJtYXhfeSI6MTA4MCwibWluX3giOjAsIm1pbl95IjotMTJ9fQ" ! """{"event":"page_ping","mobile":true,"properties":{"max_x":960,"max_y":1080,"min_x":0,"min_y":-12}}""" | + "Lua Tracker JSON #4" !! "eyJldmVudCI6ImJhc2tldF9jaGFuZ2UiLCJwcmljZSI6MjMuMzksInByb2R1Y3RfaWQiOiJQQlowMDAzNDUiLCJxdWFudGl0eSI6LTIsInRzdGFtcCI6MTY3ODAyMzAwMH0" ! """{"event":"basket_change","price":23.39,"product_id":"PBZ000345","quantity":-2,"tstamp":1678023000}""" | + "JS Tracker JSON #1" !! "eyJwcm9kdWN0X2lkIjoiQVNPMDEwNDMiLCJjYXRlZ29yeSI6IkRyZXNzZXMiLCJicmFuZCI6IkFDTUUiLCJyZXR1cm5pbmciOnRydWUsInByaWNlIjo0OS45NSwic2l6ZXMiOlsieHMiLCJzIiwibCIsInhsIiwieHhsIl0sImF2YWlsYWJsZV9zaW5jZSRkdCI6MTU4MDF9" ! """{"product_id":"ASO01043","category":"Dresses","brand":"ACME","returning":true,"price":49.95,"sizes":["xs","s","l","xl","xxl"],"available_since$dt":15801}""" | + "Unescaped characters" !! "äöü - &" ! "" | + "Blank string" !! "" ! "" |> { (_, str, expected) => { ConversionUtils.decodeBase64Url(FieldName, str) must beSuccessful(expected) } } } -class ValidateUuidSpec extends Specification with DataTables with ValidationMatchers with ScalaCheck { +class ValidateUuidSpec + extends Specification + with DataTables + with ValidationMatchers + with ScalaCheck { def is = s2""" This is a specification to test the validateUuid function validateUuid should return a lowercased UUID for a valid lower/upper-case UUID $e1 @@ -234,7 +247,7 @@ class ValidateUuidSpec extends Specification with DataTables with ValidationMatc val FieldName = "uuid" def e1 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "SPEC NAME" || "INPUT STR" | "EXPECTED" | "Lowercase UUID #1" !! "f732d278-120e-4ab6-845b-c1f11cd85dc7" ! "f732d278-120e-4ab6-845b-c1f11cd85dc7" | "Lowercase UUID #2" !! "a729d278-110a-4ac6-845b-d1f12ce45ac7" ! "a729d278-110a-4ac6-845b-d1f12ce45ac7" | "Uppercase UUID #1" !! "A729D278-110A-4AC6-845B-D1F12CE45AC7" ! "a729d278-110a-4ac6-845b-d1f12ce45ac7" | @@ -251,7 +264,8 @@ class ValidateUuidSpec extends Specification with DataTables with ValidationMatc // so low that we can just use ScalaCheck here. Checks null too def e2 = check { (str: String) => - ConversionUtils.validateUuid(FieldName, str) must beFailing(s"Field [$FieldName]: [$str] is not a valid UUID") + ConversionUtils.validateUuid(FieldName, str) must beFailing( + s"Field [$FieldName]: [$str] is not a valid UUID") } } @@ -268,38 +282,38 @@ class StringToDoublelikeSpec extends Specification with DataTables with Validati input => "Field [%s]: cannot convert [%s] to Double-like String".format(FieldName, input) def e1 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Empty string" !! "" ! err("") | - "Number with commas" !! "19,999.99" ! err("19,999.99") | - "Hexadecimal number" !! "0x54" ! err("0x54") | - "Bad sci. notation" !! "-7.51E^9" ! err("-7.51E^9") | - "German number" !! "1.000,3932" ! err("1.000,3932") | - "NaN" !! "NaN" ! err("NaN") | - "English string" !! "hi & bye" ! err("hi & bye") | - "Vietnamese name" !! "Trịnh Công Sơn" ! err("Trịnh Công Sơn") |> { (_, str, expected) => + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Empty string" !! "" ! err("") | + "Number with commas" !! "19,999.99" ! err("19,999.99") | + "Hexadecimal number" !! "0x54" ! err("0x54") | + "Bad sci. notation" !! "-7.51E^9" ! err("-7.51E^9") | + "German number" !! "1.000,3932" ! err("1.000,3932") | + "NaN" !! "NaN" ! err("NaN") | + "English string" !! "hi & bye" ! err("hi & bye") | + "Vietnamese name" !! "Trịnh Công Sơn" ! err("Trịnh Công Sơn") |> { (_, str, expected) => ConversionUtils.stringToDoublelike(FieldName, str) must beFailing(expected) } def e2 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Integer #1" !! "23" ! "23" | - "Integer #2" !! "23." ! "23" | - "Negative integer" !! "-2012103" ! "-2012103" | - "Null value (raw)" !! null ! null | - "Null value (String)" !! "null" ! null | - "Arabic number" !! "٤٥٦٧.٦٧" ! "4567.67" | - "Floating point #1" !! "1999.99" ! "1999.99" | - "Floating point #2" !! "1999.00" ! "1999.00" | - "Floating point #3" !! "78694353.00001" ! "78694353.00001" | - "Floating point #4" !! "-78694353.00001" ! "-78694353.00001" | - "Sci. notation #1" !! "4.321768E3" ! "4321.768" | - "Sci. notation #2" !! "6.72E9" ! "6720000000" | - "Sci. notation #3" !! "7.51E-9" ! "0.00000000751" |> { (_, str, expected) => + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Integer #1" !! "23" ! "23" | + "Integer #2" !! "23." ! "23" | + "Negative integer" !! "-2012103" ! "-2012103" | + "Null value (raw)" !! null ! null | + "Null value (String)" !! "null" ! null | + "Arabic number" !! "٤٥٦٧.٦٧" ! "4567.67" | + "Floating point #1" !! "1999.99" ! "1999.99" | + "Floating point #2" !! "1999.00" ! "1999.00" | + "Floating point #3" !! "78694353.00001" ! "78694353.00001" | + "Floating point #4" !! "-78694353.00001" ! "-78694353.00001" | + "Sci. notation #1" !! "4.321768E3" ! "4321.768" | + "Sci. notation #2" !! "6.72E9" ! "6720000000" | + "Sci. notation #3" !! "7.51E-9" ! "0.00000000751" |> { (_, str, expected) => ConversionUtils.stringToDoublelike(FieldName, str) must beSuccessful(expected) } val BigNumber = "78694235323.00000001" // Redshift only supports 15 significant digits for a Double - def e3 = ConversionUtils.stringToDoublelike(FieldName, BigNumber) must beSuccessful(BigNumber) + def e3 = ConversionUtils.stringToDoublelike(FieldName, BigNumber) must beSuccessful(BigNumber) } @@ -311,26 +325,27 @@ class StringToJIntegerSpec extends Specification with DataTables with Validation """ val FieldName = "val" - def err: (String) => String = input => "Field [%s]: cannot convert [%s] to Int".format(FieldName, input) + def err: (String) => String = + input => "Field [%s]: cannot convert [%s] to Int".format(FieldName, input) def e1 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Empty string" !! "" ! err("") | - "Floating point #1" !! "1999." ! err("1999.") | - "Floating point #2" !! "1999.00" ! err("1999.00") | - "Hexadecimal number" !! "0x54" ! err("0x54") | - "NaN" !! "NaN" ! err("NaN") | - "Sci. notation" !! "6.72E5" ! err("6.72E5") |> { (_, str, expected) => + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Empty string" !! "" ! err("") | + "Floating point #1" !! "1999." ! err("1999.") | + "Floating point #2" !! "1999.00" ! err("1999.00") | + "Hexadecimal number" !! "0x54" ! err("0x54") | + "NaN" !! "NaN" ! err("NaN") | + "Sci. notation" !! "6.72E5" ! err("6.72E5") |> { (_, str, expected) => ConversionUtils.stringToJInteger(FieldName, str) must beFailing(expected) } def e2 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Integer #1" !! "0" ! 0 | - "Integer #2" !! "23" ! 23 | - "Negative integer #1" !! "-2012103" ! -2012103 | - "Negative integer #2" !! "-1" ! -1 | - "Null" !! null ! null |> { (_, str, expected) => + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Integer #1" !! "0" ! 0 | + "Integer #2" !! "23" ! 23 | + "Negative integer #1" !! "-2012103" ! -2012103 | + "Negative integer #2" !! "-1" ! -1 | + "Null" !! null ! null |> { (_, str, expected) => ConversionUtils.stringToJInteger(FieldName, str) must beSuccessful(expected) } } @@ -347,21 +362,21 @@ class StringToBooleanlikeJByteSpec extends Specification with DataTables with Va input => "Field [%s]: cannot convert [%s] to Boolean-like JByte".format(FieldName, input) def e1 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "Empty string" !! "" ! err("") | - "Small number" !! "2" ! err("2") | - "Negative number" !! "-1" ! err("-1") | - "Floating point number" !! "0.0" ! err("0.0") | - "Large number" !! "19,999.99" ! err("19,999.99") | - "Text #1" !! "a" ! err("a") | - "Text #2" !! "0x54" ! err("0x54") |> { (_, str, expected) => + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "Empty string" !! "" ! err("") | + "Small number" !! "2" ! err("2") | + "Negative number" !! "-1" ! err("-1") | + "Floating point number" !! "0.0" ! err("0.0") | + "Large number" !! "19,999.99" ! err("19,999.99") | + "Text #1" !! "a" ! err("a") | + "Text #2" !! "0x54" ! err("0x54") |> { (_, str, expected) => ConversionUtils.stringToBooleanlikeJByte(FieldName, str) must beFailing(expected) } def e2 = - "SPEC NAME" || "INPUT STR" | "EXPECTED" | - "True aka 1" !! "1" ! 1.toByte | - "False aka 0" !! "0" ! 0.toByte |> { (_, str, expected) => + "SPEC NAME" || "INPUT STR" | "EXPECTED" | + "True aka 1" !! "1" ! 1.toByte | + "False aka 0" !! "0" ! 0.toByte |> { (_, str, expected) => ConversionUtils.stringToBooleanlikeJByte(FieldName, str) must beSuccessful(expected) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/ShredderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/ShredderSpec.scala index e7ab1d4e7..9e9f744d9 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/ShredderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/ShredderSpec.scala @@ -24,23 +24,24 @@ class ShredderSpec extends Specification { shred should extract the JSONs from an unstructured event with multiple contexts $e2 """ - val EventId = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" + val EventId = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" val CollectorTimestamp = "2014-04-29 09:00:54.000" implicit val resolver = SpecHelpers.IgluResolver def e1 = Shredder.makePartialHierarchy(EventId, CollectorTimestamp) must_== - TypeHierarchy(rootId = EventId, - rootTstamp = CollectorTimestamp, - refRoot = "events", - refTree = List("events"), - refParent = "events") + TypeHierarchy( + rootId = EventId, + rootTstamp = CollectorTimestamp, + refRoot = "events", + refTree = List("events"), + refParent = "events") def e2 = { val event = { val e = new EnrichedEvent() - e.event_id = EventId + e.event_id = EventId e.collector_tstamp = CollectorTimestamp e.unstruct_event = """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/link_click/jsonschema/1-0-0","data":{"targetUrl":"http://snowplowanalytics.com/blog/page2","elementClasses":["next"]}}}""" diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/TypeHierarchySpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/TypeHierarchySpec.scala index c5481e94b..4b217adea 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/TypeHierarchySpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/shredder/TypeHierarchySpec.scala @@ -12,6 +12,10 @@ */ package com.snowplowanalytics.snowplow.enrich.common.utils.shredder +import cats.syntax.either._ +import io.circe.generic.auto._ +import io.circe.parser._ +import io.circe.syntax._ import org.specs2.Specification class TypeHierarchySpec extends Specification { @@ -21,29 +25,39 @@ class TypeHierarchySpec extends Specification { the complete method should finalize a partial TypeHierarchy $e2 """ - val EventId = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" + val EventId = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" val CollectorTimestamp = "2014-04-29 09:00:54.000" def e1 = { val hierarchy = - TypeHierarchy(rootId = EventId, - rootTstamp = CollectorTimestamp, - refRoot = "events", - refTree = List("events", "new_ticket"), - refParent = "events") + TypeHierarchy( + rootId = EventId, + rootTstamp = CollectorTimestamp, + refRoot = "events", + refTree = List("events", "new_ticket"), + refParent = "events") + + val json = parse(s"""{ + "rootId": "$EventId", + "rootTstamp": "$CollectorTimestamp", + "refRoot": "events", + "refTree": ["events", "new_ticket"], + "refParent": "events" + }""").toOption.get // TODO: add missing refTree - hierarchy.toJsonNode.toString must_== s"""{"rootId":"${EventId}","rootTstamp":"${CollectorTimestamp}","refRoot":"events","refTree":["events","new_ticket"],"refParent":"events"}""" + hierarchy.asJson must_== json } def e2 = { val partial = Shredder.makePartialHierarchy(EventId, CollectorTimestamp) partial.complete(List("link_click", "elementClasses")) must_== - TypeHierarchy(rootId = EventId, - rootTstamp = CollectorTimestamp, - refRoot = "events", - refTree = List("events", "link_click", "elementClasses"), - refParent = "link_click") + TypeHierarchy( + rootId = EventId, + rootTstamp = CollectorTimestamp, + refRoot = "events", + refTree = List("events", "link_click", "elementClasses"), + refParent = "link_click") } } diff --git a/project/CommonDependencies.scala b/project/CommonDependencies.scala index 0a6a6440b..79e3ca963 100644 --- a/project/CommonDependencies.scala +++ b/project/CommonDependencies.scala @@ -18,8 +18,6 @@ import sbt._ object Dependencies { val resolutionRepos = Seq( - // Required for our json4s snapshot - "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/", // For some misc Scalding and Twitter libs "Concurrent Maven Repo" at "http://conjars.org/repo", // For Twitter's util functions @@ -58,7 +56,9 @@ object Dependencies { val schemaSniffer = "0.0.0" val refererParser = "0.3.1" val maxmindIplookups = "0.4.0" - val json4s = "3.2.11" + val circe = "0.11.1" + val circeOptics = "0.11.0" + val circeJackson = "0.11.1" val igluClient = "0.5.0" val scalaForex = "0.5.0" val scalaWeather = "0.3.0" @@ -95,6 +95,14 @@ object Dependencies { val guava = "com.google.guava" % "guava" % V.guava // Scala + val circeDeps = List( + "circe-core", + "circe-generic", + "circe-parser", + "circe-literal" + ).map("io.circe" %% _ % V.circe) + val circeOptics = "io.circe" %% "circe-optics" % V.circeOptics + val circeJackson = "io.circe" %% "circe-jackson28" % V.circeJackson val scalaForex = "com.snowplowanalytics" %% "scala-forex" % V.scalaForex val scalaz7 = "org.scalaz" %% "scalaz-core" % V.scalaz7 val snowplowRawEvent = "com.snowplowanalytics" % "snowplow-thrift-raw-event" % V.snowplowRawEvent @@ -102,9 +110,7 @@ object Dependencies { val schemaSniffer = "com.snowplowanalytics" % "schema-sniffer-1" % V.schemaSniffer val refererParser = "com.snowplowanalytics" %% "referer-parser" % V.refererParser val maxmindIplookups = "com.snowplowanalytics" %% "scala-maxmind-iplookups" % V.maxmindIplookups - val json4sJackson = "org.json4s" %% "json4s-jackson" % V.json4s - val json4sScalaz = "org.json4s" %% "json4s-scalaz" % V.json4s - val igluClient = "com.snowplowanalytics" %% "iglu-scala-client" % V.igluClient + val igluClient = "com.snowplowanalytics" %% "iglu-scala-client" % V.igluClient val scalaUri = "io.lemonlabs" %% "scala-uri" % V.scalaUri val scalaWeather = "com.snowplowanalytics" %% "scala-weather" % V.scalaWeather val scalaj = "org.scalaj" %% "scalaj-http" % V.scalaj @@ -114,6 +120,6 @@ object Dependencies { val scalazSpecs2 = "org.typelevel" %% "scalaz-specs2" % V.scalazSpecs2 % "test" val scalaCheck = "org.scalacheck" %% "scalacheck" % V.scalaCheck % "test" val scaldingArgs = "com.twitter" %% "scalding-args" % V.scaldingArgs % "test" - val mockito = "org.mockito" % "mockito-core" % V.mockito % "test" + val mockito = "org.mockito" % "mockito-core" % V.mockito % "test" } }