diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/ParsingError.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/ParsingError.scala new file mode 100644 index 0000000..b4c3931 --- /dev/null +++ b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/ParsingError.scala @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.analytics.scalasdk + +import cats.data.NonEmptyList +import cats.syntax.either._ +import io.circe._ +import io.circe.syntax._ +import com.snowplowanalytics.snowplow.analytics.scalasdk.decode.Key + +/** + * Represents error during parsing TSV event + */ +sealed trait ParsingError + +object ParsingError { + + /** + * Represents error which given line is not a TSV + */ + final case object NonTSVPayload extends ParsingError + + /** + * Represents error which number of given columns is not equal + * to number of expected columns + * @param columnCount mismatched column count in the event + */ + final case class ColumnNumberMismatch(columnCount: Int) extends ParsingError + + /** + * Represents error which encountered while decoding values in row + * @param errors Infos of errors which encountered during decoding + */ + final case class RowDecodingError(errors: NonEmptyList[RowDecodingErrorInfo]) extends ParsingError + + /** + * Gives info about the reasons of the errors during decoding value in row + */ + sealed trait RowDecodingErrorInfo + + object RowDecodingErrorInfo { + /** + * Represents cases where value in a field is not valid + * e.g invalid timestamp, invalid UUID + * @param key key of field + * @param value value of field + * @param message error message + */ + final case class InvalidValue(key: Key, value: String, message: String) extends RowDecodingErrorInfo + + /** + * Represents cases which getting error is not expected while decoding row + * For example, while parsing the list of tuples to HList in the + * RowDecoder, getting more or less values than expected is impossible + * due to type check. Therefore 'UnexpectedRowDecodingError' is returned for + * these cases. These errors can be ignored since they are not possible to get + * @param error error message + */ + final case class UnexpectedRowDecodingError(error: String) extends RowDecodingErrorInfo + + implicit val analyticsSdkRowDecodingErrorInfoCirceEncoder: Encoder[RowDecodingErrorInfo] = + Encoder.instance { + case InvalidValue(key, value, message) => + Json.obj( + "type" := "InvalidValue", + "key" := key, + "value" := value, + "message" := message + ) + case UnexpectedRowDecodingError(error: String) => + Json.obj( + "type" := "UnexpectedRowDecodingError", + "error" := error + ) + } + + implicit val analyticsSdkRowDecodingErrorInfoCirceDecoder: Decoder[RowDecodingErrorInfo] = + Decoder.instance { cursor => + for { + errorType <- cursor.downField("type").as[String] + result <- errorType match { + case "InvalidValue" => + for { + key <- cursor.downField("key").as[Key] + value <- cursor.downField("value").as[String] + message <- cursor.downField("message").as[String] + } yield InvalidValue(key, value, message) + + case "UnexpectedRowDecodingError" => + cursor + .downField("error") + .as[String] + .map(UnexpectedRowDecodingError) + } + } yield result + } + + implicit val analyticsSdkKeyCirceEncoder: Encoder[Key] = + Encoder.instance(_.toString.stripPrefix("'").asJson) + + implicit val analyticsSdkKeyCirceDecoder: Decoder[Key] = + Decoder.instance(_.as[String].map(Symbol(_))) + + } + + implicit val analyticsSdkParsingErrorCirceEncoder: Encoder[ParsingError] = + Encoder.instance { + case NonTSVPayload => + Json.obj("type" := "NonTSVPayload") + case ColumnNumberMismatch(columnCount) => + Json.obj( + "type" := "ColumnNumberMismatch", + "columnCount" := columnCount + ) + case RowDecodingError(errors) => + Json.obj( + "type" := "RowDecodingError", + "errors" := errors.asJson + ) + } + + implicit val analyticsSdkParsingErrorCirceDecoder: Decoder[ParsingError] = + Decoder.instance { cursor => + for { + error <- cursor.downField("type").as[String] + result <- error match { + case "NonTSVPayload" => + NonTSVPayload.asRight + case "ColumnNumberMismatch" => + cursor + .downField("columnCount") + .as[Int] + .map(ColumnNumberMismatch) + case "RowDecodingError" => + cursor + .downField("errors") + .as[NonEmptyList[RowDecodingErrorInfo]] + .map(RowDecodingError) + case _ => + DecodingFailure( + s"Error type $error cannot be recognized as Analytics SDK Parsing Error", + cursor.history).asLeft + } + } yield result + } +} diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/Parser.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/Parser.scala index 2b88f02..a8d5868 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/Parser.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/Parser.scala @@ -15,8 +15,8 @@ package com.snowplowanalytics.snowplow.analytics.scalasdk.decode import shapeless._ import shapeless.ops.record._ import shapeless.ops.hlist._ - -import Parser._ +import cats.data.{NonEmptyList, Validated} +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.{ColumnNumberMismatch, NonTSVPayload, RowDecodingError} private[scalasdk] trait Parser[A] extends Serializable { /** Heterogeneous TSV values */ @@ -33,9 +33,16 @@ private[scalasdk] trait Parser[A] extends Serializable { def parse(row: String): DecodeResult[A] = { val values = row.split("\t", -1) - val zipped = knownKeys.zipAll(values, UnknownKeyPlaceholder, ValueIsMissingPlaceholder) - val decoded = decoder(zipped) - decoded.map { decodedValue => generic.from(decodedValue) } + if (values.length == 1) { + Validated.Invalid(NonTSVPayload) + } + else if (values.length != knownKeys.length) { + Validated.Invalid(ColumnNumberMismatch(values.length)) + } else { + val zipped = knownKeys.zip(values) + val decoded = decoder(zipped).leftMap(e => RowDecodingError(e)) + decoded.map { decodedValue => generic.from(decodedValue) } + } } } @@ -61,11 +68,6 @@ object Parser { } } - /** Key name that will be used if TSV has more columns than a class */ - val UnknownKeyPlaceholder = 'UnknownKey - /** Value that will be used if class has more fields than a TSV */ - val ValueIsMissingPlaceholder = "VALUE IS MISSING" - /** Derive a TSV parser for `A` */ private[scalasdk] def deriveFor[A]: DeriveParser[A] = new DeriveParser[A] {} diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/RowDecoder.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/RowDecoder.scala index 91ddb45..1aaaeaf 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/RowDecoder.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/RowDecoder.scala @@ -16,6 +16,7 @@ import shapeless._ import cats.syntax.validated._ import cats.syntax.either._ import cats.syntax.apply._ +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo.UnexpectedRowDecodingError /** * Type class to decode List of keys-value pairs into HList @@ -23,7 +24,7 @@ import cats.syntax.apply._ * Values are actual TSV columns */ private[scalasdk] trait RowDecoder[L <: HList] extends Serializable { - def apply(row: List[(Key, String)]): DecodeResult[L] + def apply(row: List[(Key, String)]): RowDecodeResult[L] } private[scalasdk] object RowDecoder { @@ -31,25 +32,24 @@ private[scalasdk] object RowDecoder { def apply[L <: HList](implicit fromRow: RowDecoder[L]): RowDecoder[L] = fromRow - def fromFunc[L <: HList](f: List[(Key, String)] => DecodeResult[L]): RowDecoder[L] = + def fromFunc[L <: HList](f: List[(Key, String)] => RowDecodeResult[L]): RowDecoder[L] = new RowDecoder[L] { def apply(row: List[(Key, String)]) = f(row) } /** Parse TSV row into HList */ - private def parse[H: ValueDecoder, T <: HList: RowDecoder](row: List[(Key, String)]) = + private def parse[H: ValueDecoder, T <: HList: RowDecoder](row: List[(Key, String)]): RowDecodeResult[H :: T] = row match { case h :: t => - val hv: DecodeResult[H] = - ValueDecoder[H].parse(h).leftMap(_._2).toValidatedNel - val tv = RowDecoder[T].apply(t) + val hv: RowDecodeResult[H] = ValueDecoder[H].parse(h).toValidatedNel + val tv: RowDecodeResult[T] = RowDecoder[T].apply(t) (hv, tv).mapN { _ :: _ } - case Nil => "Not enough values, format is invalid".invalidNel + case Nil => UnexpectedRowDecodingError("Not enough values, format is invalid").invalidNel } - implicit val hnilFromRow: RowDecoder[HNil] = fromFunc { + implicit def hnilFromRow: RowDecoder[HNil] = fromFunc { case Nil => HNil.validNel - case rows => s"No more values expected, following provided: ${rows.map(_._2).mkString(", ")}".invalidNel + case rows => UnexpectedRowDecodingError(s"No more values expected, following provided: ${rows.map(_._2).mkString(", ")}").invalidNel } implicit def hconsFromRow[H: ValueDecoder, T <: HList: RowDecoder]: RowDecoder[H :: T] = diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala index 4e05a50..b28f8b7 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala @@ -34,6 +34,8 @@ import io.circe.{Error, Json} // This library import com.snowplowanalytics.snowplow.analytics.scalasdk.Common.{ContextsCriterion, UnstructEventCriterion} import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent.{Contexts, UnstructEvent} +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo._ private[decode] trait ValueDecoder[A] { def parse(column: (Key, String)): DecodedValue[A] @@ -50,7 +52,7 @@ private[decode] object ValueDecoder { implicit final val stringColumnDecoder: ValueDecoder[String] = fromFunc[String] { case (key, value) => - if (value.isEmpty) (key, s"Field $key cannot be empty").asLeft else value.asRight + if (value.isEmpty) InvalidValue(key, value, s"Field $key cannot be empty").asLeft else value.asRight } implicit final val stringOptionColumnDecoder: ValueDecoder[Option[String]] = @@ -67,7 +69,7 @@ private[decode] object ValueDecoder { value.toInt.some.asRight } catch { case _: NumberFormatException => - (key, s"Cannot parse key $key with value $value into integer").asLeft + InvalidValue(key, value, s"Cannot parse key $key with value $value into integer").asLeft } } @@ -75,13 +77,13 @@ private[decode] object ValueDecoder { fromFunc[UUID] { case (key, value) => if (value.isEmpty) - (key, s"Field $key cannot be empty").asLeft + InvalidValue(key, value, s"Field $key cannot be empty").asLeft else try { - UUID.fromString(value).asRight[(Key, String)] + UUID.fromString(value).asRight[RowDecodingErrorInfo] } catch { case _: IllegalArgumentException => - (key, s"Cannot parse key $key with value $value into UUID").asLeft + InvalidValue(key, value, s"Cannot parse key $key with value $value into UUID").asLeft } } @@ -92,7 +94,7 @@ private[decode] object ValueDecoder { case "0" => false.some.asRight case "1" => true.some.asRight case "" => none[Boolean].asRight - case _ => (key, s"Cannot parse key $key with value $value into boolean").asLeft + case _ => InvalidValue(key, value, s"Cannot parse key $key with value $value into boolean").asLeft } } @@ -105,7 +107,7 @@ private[decode] object ValueDecoder { value.toDouble.some.asRight } catch { case _: NumberFormatException => - (key, s"Cannot parse key $key with value $value into double").asLeft + InvalidValue(key, value, s"Cannot parse key $key with value $value into double").asLeft } } @@ -113,14 +115,14 @@ private[decode] object ValueDecoder { fromFunc[Instant] { case (key, value) => if (value.isEmpty) - (key, s"Field $key cannot be empty").asLeft + InvalidValue(key, value, s"Field $key cannot be empty").asLeft else { val tstamp = reformatTstamp(value) try { Instant.parse(tstamp).asRight } catch { case _: DateTimeParseException => - (key, s"Cannot parse key $key with value $value into datetime").asLeft + InvalidValue(key, value, s"Cannot parse key $key with value $value into datetime").asLeft } } } @@ -129,14 +131,14 @@ private[decode] object ValueDecoder { fromFunc[Option[Instant]] { case (key, value) => if (value.isEmpty) - none[Instant].asRight[(Key, String)] + none[Instant].asRight[RowDecodingErrorInfo] else { val tstamp = reformatTstamp(value) try { Instant.parse(tstamp).some.asRight } catch { case _: DateTimeParseException => - (key, s"Cannot parse key $key with value $value into datetime").asLeft + InvalidValue(key, value, s"Cannot parse key $key with value $value into datetime").asLeft } } } @@ -144,9 +146,9 @@ private[decode] object ValueDecoder { implicit final val unstructuredJson: ValueDecoder[UnstructEvent] = fromFunc[UnstructEvent] { case (key, value) => - def asLeft(error: Error): (Key, String) = (key, error.show) + def asLeft(error: Error): RowDecodingErrorInfo = InvalidValue(key, value, error.show) if (value.isEmpty) - UnstructEvent(None).asRight[(Key, String)] + UnstructEvent(None).asRight[RowDecodingErrorInfo] else parseJson(value) .flatMap(_.as[SelfDescribingData[Json]]) @@ -154,7 +156,7 @@ private[decode] object ValueDecoder { case Right(SelfDescribingData(schema, data)) if UnstructEventCriterion.matches(schema) => data.as[SelfDescribingData[Json]].leftMap(asLeft).map(_.some).map(UnstructEvent.apply) case Right(SelfDescribingData(schema, _)) => - (key, s"Unknown payload: ${schema.toSchemaUri}").asLeft[UnstructEvent] + InvalidValue(key, value, s"Unknown payload: ${schema.toSchemaUri}").asLeft[UnstructEvent] case Left(error) => error.asLeft[UnstructEvent] } } @@ -162,9 +164,9 @@ private[decode] object ValueDecoder { implicit final val contexts: ValueDecoder[Contexts] = fromFunc[Contexts] { case (key, value) => - def asLeft(error: Error): (Key, String) = (key, error.show) + def asLeft(error: Error): RowDecodingErrorInfo = InvalidValue(key, value, error.show) if (value.isEmpty) - Contexts(List()).asRight[(Key, String)] + Contexts(List()).asRight[RowDecodingErrorInfo] else parseJson(value) .flatMap(_.as[SelfDescribingData[Json]]) @@ -172,7 +174,7 @@ private[decode] object ValueDecoder { case Right(SelfDescribingData(schema, data)) if ContextsCriterion.matches(schema) => data.as[List[SelfDescribingData[Json]]].leftMap(asLeft).map(Contexts.apply) case Right(SelfDescribingData(schema, _)) => - (key, s"Unknown payload: ${schema.toSchemaUri}").asLeft[Contexts] + InvalidValue(key, value, s"Unknown payload: ${schema.toSchemaUri}").asLeft[Contexts] case Left(error) => error.asLeft[Contexts] } } diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/package.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/package.scala index 7027446..88c9225 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/package.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/package.scala @@ -12,15 +12,19 @@ */ package com.snowplowanalytics.snowplow.analytics.scalasdk -import cats.data.ValidatedNel +import cats.data.{Validated, ValidatedNel} +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo package object decode { /** Expected name of the field */ type Key = Symbol /** Result of single-value parsing */ - type DecodedValue[A] = Either[(Key, String), A] + type DecodedValue[A] = Either[RowDecodingErrorInfo, A] - /** Result of TSV line parsing, which is either an event or non empty list of parse errors */ - type DecodeResult[A] = ValidatedNel[String, A] + /** Result of row decode process */ + type RowDecodeResult[A] = ValidatedNel[RowDecodingErrorInfo, A] + + /** Result of TSV line parsing, which is either an event or parse error */ + type DecodeResult[A] = Validated[ParsingError, A] } diff --git a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala index 9fc4b0a..ff29e59 100644 --- a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala +++ b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala @@ -34,6 +34,8 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData // This library import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent._ +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError._ +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo._ /** * Tests Event case class @@ -713,7 +715,7 @@ class EventSpec extends Specification { "event_version": "1-0-0", "event_fingerprint": "e3dbfa9cca0412c3d4052863cefb547f", "true_tstamp": "2013-11-26T00:03:57.886Z" - }""").getOrElse(throw new RuntimeException("Failed to parse expected JSON")) + }""").right.getOrElse(throw new RuntimeException("Failed to parse expected JSON")) // JSON output must be equal to output from the old transformer. (NB: field ordering in new JSON will be randomized) eventJson mustEqual expectedJson @@ -1127,7 +1129,7 @@ class EventSpec extends Specification { "event_version": "1-0-0", "event_fingerprint": "e3dbfa9cca0412c3d4052863cefb547f", "true_tstamp": "2013-11-26T00:03:57.886Z" - }""").getOrElse(throw new RuntimeException("Failed to parse expected JSON")) + }""").right.getOrElse(throw new RuntimeException("Failed to parse expected JSON")) // JSON output must be equal to output from the old transformer. (NB: field ordering in new JSON will be randomized) eventJson mustEqual expectedJson @@ -1658,7 +1660,7 @@ class EventSpec extends Specification { "event_version": "1-0-0", "event_fingerprint": "e3dbfa9cca0412c3d4052863cefb547f", "true_tstamp": "2013-11-26T00:03:57.886Z" - }""").getOrElse(throw new RuntimeException("Failed to parse expected JSON")) + }""").right.getOrElse(throw new RuntimeException("Failed to parse expected JSON")) // JSON output must be equal to output from the old transformer. (NB: field ordering in new JSON will be randomized) eventJson mustEqual expectedJson @@ -1986,23 +1988,187 @@ class EventSpec extends Specification { "event_name" -> "link_click", "event_format" -> "jsonschema", "event_version" -> "1-0-0", - "event_fingerprint" -> "e3dbfa9cca0412c3d4052863cefb547f" + "event_fingerprint" -> "e3dbfa9cca0412c3d4052863cefb547f", + "true_tstamp" -> "2013-11-26 00:03:57.886" ) val eventValues = input.unzip._2.mkString("\t") val event = Event.parse(eventValues) // Case class must be correctly invalidated - event mustEqual Invalid(NonEmptyList.of( - "Cannot parse key 'etl_tstamp with value not_an_instant into datetime", - "Field 'collector_tstamp cannot be empty", - "Cannot parse key 'event_id with value not_a_uuid into UUID", - "Cannot parse key 'txn_id with value not_an_integer into integer", - "Field 'v_collector cannot be empty", - "Cannot parse key 'geo_latitude with value not_a_double into double", - "Cannot parse key 'br_features_pdf with value not_a_boolean into boolean", - "Cannot parse key 'true_tstamp with value VALUE IS MISSING into datetime" - )) + val res = RowDecodingError( + NonEmptyList.of( + InvalidValue(Symbol("etl_tstamp"), "not_an_instant", "Cannot parse key 'etl_tstamp with value not_an_instant into datetime"), + InvalidValue(Symbol("collector_tstamp"), "", "Field 'collector_tstamp cannot be empty"), + InvalidValue(Symbol("event_id"), "not_a_uuid", "Cannot parse key 'event_id with value not_a_uuid into UUID"), + InvalidValue(Symbol("txn_id"), "not_an_integer", "Cannot parse key 'txn_id with value not_an_integer into integer"), + InvalidValue(Symbol("v_collector"), "", "Field 'v_collector cannot be empty"), + InvalidValue(Symbol("geo_latitude"), "not_a_double", "Cannot parse key 'geo_latitude with value not_a_double into double"), + InvalidValue(Symbol("br_features_pdf"), "not_a_boolean", "Cannot parse key 'br_features_pdf with value not_a_boolean into boolean") + ) + ) + event mustEqual Invalid(res) + } + + "fail if payload is not TSV" in { + val event = Event.parse("non tsv") + event mustEqual Invalid(NonTSVPayload) + } + + "fail if there are more column than expected" in { + val input = List( + "app_id" -> "angry-birds", + "platform" -> "web", + "etl_tstamp" -> "not_an_instant", + "collector_tstamp" -> "", + "dvce_created_tstamp" -> "2013-11-26 00:03:57.885", + "event" -> "page_view", + "event_id" -> "not_a_uuid", + "txn_id" -> "not_an_integer", + "name_tracker" -> "cloudfront-1", + "v_tracker" -> "js-2.1.0", + "v_collector" -> "", + "v_etl" -> "serde-0.5.2", + "user_id" -> "jon.doe@email.com", + "user_ipaddress" -> "92.231.54.234", + "user_fingerprint" -> "2161814971", + "domain_userid" -> "bc2e92ec6c204a14", + "domain_sessionidx" -> "3", + "network_userid" -> "ecdff4d0-9175-40ac-a8bb-325c49733607", + "geo_country" -> "US", + "geo_region" -> "TX", + "geo_city" -> "New York", + "geo_zipcode" -> "94109", + "geo_latitude" -> "not_a_double", + "geo_longitude" -> "-122.4124", + "geo_region_name" -> "Florida", + "ip_isp" -> "FDN Communications", + "ip_organization" -> "Bouygues Telecom", + "ip_domain" -> "nuvox.net", + "ip_netspeed" -> "Cable/DSL", + "page_url" -> "http://www.snowplowanalytics.com", + "page_title" -> "On Analytics", + "page_referrer" -> "", + "page_urlscheme" -> "http", + "page_urlhost" -> "www.snowplowanalytics.com", + "page_urlport" -> "80", + "page_urlpath" -> "/product/index.html", + "page_urlquery" -> "id=GTM-DLRG", + "page_urlfragment" -> "4-conclusion", + "refr_urlscheme" -> "", + "refr_urlhost" -> "", + "refr_urlport" -> "", + "refr_urlpath" -> "", + "refr_urlquery" -> "", + "refr_urlfragment" -> "", + "refr_medium" -> "", + "refr_source" -> "", + "refr_term" -> "", + "mkt_medium" -> "", + "mkt_source" -> "", + "mkt_term" -> "", + "mkt_content" -> "", + "mkt_campaign" -> "", + "contexts" -> contextsJson, + "se_category" -> "", + "se_action" -> "", + "se_label" -> "", + "se_property" -> "", + "se_value" -> "", + "unstruct_event" -> unstructJson, + "tr_orderid" -> "", + "tr_affiliation" -> "", + "tr_total" -> "", + "tr_tax" -> "", + "tr_shipping" -> "", + "tr_city" -> "", + "tr_state" -> "", + "tr_country" -> "", + "ti_orderid" -> "", + "ti_sku" -> "", + "ti_name" -> "", + "ti_category" -> "", + "ti_price" -> "", + "ti_quantity" -> "", + "pp_xoffset_min" -> "", + "pp_xoffset_max" -> "", + "pp_yoffset_min" -> "", + "pp_yoffset_max" -> "", + "useragent" -> "", + "br_name" -> "", + "br_family" -> "", + "br_version" -> "", + "br_type" -> "", + "br_renderengine" -> "", + "br_lang" -> "", + "br_features_pdf" -> "not_a_boolean", + "br_features_flash" -> "0", + "br_features_java" -> "", + "br_features_director" -> "", + "br_features_quicktime" -> "", + "br_features_realplayer" -> "", + "br_features_windowsmedia" -> "", + "br_features_gears" -> "", + "br_features_silverlight" -> "", + "br_cookies" -> "", + "br_colordepth" -> "", + "br_viewwidth" -> "", + "br_viewheight" -> "", + "os_name" -> "", + "os_family" -> "", + "os_manufacturer" -> "", + "os_timezone" -> "", + "dvce_type" -> "", + "dvce_ismobile" -> "", + "dvce_screenwidth" -> "", + "dvce_screenheight" -> "", + "doc_charset" -> "", + "doc_width" -> "", + "doc_height" -> "", + "tr_currency" -> "", + "tr_total_base" -> "", + "tr_tax_base" -> "", + "tr_shipping_base" -> "", + "ti_currency" -> "", + "ti_price_base" -> "", + "base_currency" -> "", + "geo_timezone" -> "", + "mkt_clickid" -> "", + "mkt_network" -> "", + "etl_tags" -> "", + "dvce_sent_tstamp" -> "", + "refr_domain_userid" -> "", + "refr_dvce_tstamp" -> "", + "derived_contexts" -> derivedContextsJson, + "domain_sessionid" -> "2b15e5c8-d3b1-11e4-b9d6-1681e6b88ec1", + "derived_tstamp" -> "2013-11-26 00:03:57.886", + "event_vendor" -> "com.snowplowanalytics.snowplow", + "event_name" -> "link_click", + "event_format" -> "jsonschema", + "event_version" -> "1-0-0", + "event_fingerprint" -> "e3dbfa9cca0412c3d4052863cefb547f", + "true_tstamp" -> "2013-11-26 00:03:57.886", + "additional_field" -> "mock_value" + ) + + val eventValues = input.unzip._2.mkString("\t") + val event = Event.parse(eventValues) + + event mustEqual Invalid(ColumnNumberMismatch(132)) + } + + "fail if there are less column than expected" in { + val input = List( + "app_id" -> "angry-birds", + "platform" -> "web", + "etl_tstamp" -> "not_an_instant", + "collector_tstamp" -> "" + ) + + val eventValues = input.unzip._2.mkString("\t") + val event = Event.parse(eventValues) + + event mustEqual Invalid(ColumnNumberMismatch(4)) } "successfully decode encoded event which has no contexts or unstruct_event" in { diff --git a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/ParsingErrorSpec.scala b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/ParsingErrorSpec.scala new file mode 100644 index 0000000..c206ca7 --- /dev/null +++ b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/ParsingErrorSpec.scala @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.analytics.scalasdk + +import cats.data.NonEmptyList + +import io.circe.{Decoder, Json} +import io.circe.syntax._ +import io.circe.parser._ + +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError._ +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo._ +import org.specs2.Specification + +class ParsingErrorSpec extends Specification { def is = s2""" + ParsingError encoder-decoder + works correctly with NonTSVPayload error $e1 + works correctly with ColumnNumberMismatch error $e2 + works correctly with RowDecodingError $e3 + """ + + def e1 = { + val errorJson = parseJson( + """ + |{ + | "type": "NonTSVPayload" + |} + """.stripMargin + ) + + val decoded = decodeJson[ParsingError](errorJson) + val encoded = decoded.asJson + + (decoded must beEqualTo(NonTSVPayload)) and (encoded must beEqualTo(errorJson)) + } + + def e2 = { + val errorJson = parseJson( + """ + |{ + | "type": "ColumnNumberMismatch", + | "columnCount": 120 + |} + """.stripMargin + ) + + val decoded = decodeJson[ParsingError](errorJson) + val encoded = decoded.asJson + + (decoded must beEqualTo(ColumnNumberMismatch(120))) and (encoded must beEqualTo(errorJson)) + } + + def e3 = { + val errorJson = parseJson( + """ + |{ + | "type": "RowDecodingError", + | "errors": [ + | { + | "type": "InvalidValue", + | "key": "exampleKey", + | "value": "exampleValue", + | "message": "exampleMessage" + | }, + | { + | "type": "UnexpectedRowDecodingError", + | "error": "exampleError" + | } + | ] + |} + """.stripMargin + ) + + val decoded = decodeJson[ParsingError](errorJson) + val encoded = decoded.asJson + + val expected = RowDecodingError( + NonEmptyList.of( + InvalidValue(Symbol("exampleKey"), "exampleValue", "exampleMessage"), + UnexpectedRowDecodingError("exampleError") + ) + ) + + (decoded must beEqualTo(expected)) and (encoded must beEqualTo(errorJson)) + } + + private def parseJson(jsonStr: String): Json = + parse(jsonStr).right.getOrElse(throw new RuntimeException("Failed to parse expected JSON")) + + private def decodeJson[A: Decoder](json: Json): A = { + json.as[A].right.getOrElse(throw new RuntimeException("Failed to decode to ParsingError")) + } +} diff --git a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoderSpec.scala b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoderSpec.scala index d62377a..b7d602b 100644 --- a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoderSpec.scala +++ b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoderSpec.scala @@ -33,6 +33,7 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData // This library import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent.{Contexts, UnstructEvent} +import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo._ /** * Tests ValueDecoder class @@ -41,7 +42,7 @@ class ValueDecoderSpec extends Specification { "The ValueDecoder class" should { "parse String and Option[String] values" in { - ValueDecoder[String].parse(Symbol("key"), "") mustEqual (Symbol("key"), "Field 'key cannot be empty").asLeft + ValueDecoder[String].parse(Symbol("key"), "") mustEqual InvalidValue(Symbol("key"), "", "Field 'key cannot be empty").asLeft ValueDecoder[String].parse(Symbol("key"), "value") mustEqual "value".asRight ValueDecoder[Option[String]].parse(Symbol("key"), "") mustEqual None.asRight ValueDecoder[Option[String]].parse(Symbol("key"), "value") mustEqual Some("value").asRight @@ -50,35 +51,35 @@ class ValueDecoderSpec extends Specification { "parse Option[Int] values" in { ValueDecoder[Option[Int]].parse(Symbol("key"), "") mustEqual None.asRight ValueDecoder[Option[Int]].parse(Symbol("key"), "42") mustEqual Some(42).asRight - ValueDecoder[Option[Int]].parse(Symbol("key"), "value") mustEqual (Symbol("key"), "Cannot parse key 'key with value value into integer").asLeft + ValueDecoder[Option[Int]].parse(Symbol("key"), "value") mustEqual InvalidValue(Symbol("key"), "value", "Cannot parse key 'key with value value into integer").asLeft } "parse UUID values" in { - ValueDecoder[UUID].parse(Symbol("key"), "") mustEqual (Symbol("key"), "Field 'key cannot be empty").asLeft + ValueDecoder[UUID].parse(Symbol("key"), "") mustEqual InvalidValue(Symbol("key"), "", "Field 'key cannot be empty").asLeft ValueDecoder[UUID].parse(Symbol("key"), "d2161fd1-ffed-41df-ac3e-a729012105f5") mustEqual UUID.fromString("d2161fd1-ffed-41df-ac3e-a729012105f5").asRight - ValueDecoder[UUID].parse(Symbol("key"), "value") mustEqual (Symbol("key"), "Cannot parse key 'key with value value into UUID").asLeft + ValueDecoder[UUID].parse(Symbol("key"), "value") mustEqual InvalidValue(Symbol("key"), "value" ,"Cannot parse key 'key with value value into UUID").asLeft } "parse Option[Boolean] values" in { ValueDecoder[Option[Boolean]].parse(Symbol("key"), "") mustEqual None.asRight ValueDecoder[Option[Boolean]].parse(Symbol("key"), "0") mustEqual Some(false).asRight ValueDecoder[Option[Boolean]].parse(Symbol("key"), "1") mustEqual Some(true).asRight - ValueDecoder[Option[Boolean]].parse(Symbol("key"), "value") mustEqual (Symbol("key"), "Cannot parse key 'key with value value into boolean").asLeft + ValueDecoder[Option[Boolean]].parse(Symbol("key"), "value") mustEqual InvalidValue(Symbol("key"), "value", "Cannot parse key 'key with value value into boolean").asLeft } "parse Option[Double] values" in { ValueDecoder[Option[Double]].parse(Symbol("key"), "") mustEqual None.asRight ValueDecoder[Option[Double]].parse(Symbol("key"), "42.5") mustEqual Some(42.5).asRight - ValueDecoder[Option[Double]].parse(Symbol("key"), "value") mustEqual (Symbol("key"), "Cannot parse key 'key with value value into double").asLeft + ValueDecoder[Option[Double]].parse(Symbol("key"), "value") mustEqual InvalidValue(Symbol("key"), "value", "Cannot parse key 'key with value value into double").asLeft } "parse Instant and Option[Instant] values" in { - ValueDecoder[Instant].parse(Symbol("key"), "") mustEqual (Symbol("key"), "Field 'key cannot be empty").asLeft + ValueDecoder[Instant].parse(Symbol("key"), "") mustEqual InvalidValue(Symbol("key"), "", "Field 'key cannot be empty").asLeft ValueDecoder[Instant].parse(Symbol("key"), "2013-11-26 00:03:57.885") mustEqual Instant.parse("2013-11-26T00:03:57.885Z").asRight - ValueDecoder[Instant].parse(Symbol("key"), "value") mustEqual (Symbol("key"), "Cannot parse key 'key with value value into datetime").asLeft + ValueDecoder[Instant].parse(Symbol("key"), "value") mustEqual InvalidValue(Symbol("key"), "value", "Cannot parse key 'key with value value into datetime").asLeft ValueDecoder[Option[Instant]].parse(Symbol("key"), "") mustEqual None.asRight ValueDecoder[Option[Instant]].parse(Symbol("key"), "2013-11-26 00:03:57.885") mustEqual Some(Instant.parse("2013-11-26T00:03:57.885Z")).asRight - ValueDecoder[Option[Instant]].parse(Symbol("key"), "value") mustEqual (Symbol("key"), "Cannot parse key 'key with value value into datetime").asLeft + ValueDecoder[Option[Instant]].parse(Symbol("key"), "value") mustEqual InvalidValue(Symbol("key"), "value", "Cannot parse key 'key with value value into datetime").asLeft } "parse Contexts values" in { @@ -153,7 +154,7 @@ class ValueDecoderSpec extends Specification { ) ) ).asRight - ValueDecoder[Contexts].parse(Symbol("key"), invalidPayloadContexts) mustEqual (Symbol("key"), "Unknown payload: iglu:invalid/schema/jsonschema/1-0-0").asLeft + ValueDecoder[Contexts].parse(Symbol("key"), invalidPayloadContexts) mustEqual InvalidValue(Symbol("key"), invalidPayloadContexts, "Unknown payload: iglu:invalid/schema/jsonschema/1-0-0").asLeft } "parse UnstructEvent values" in { @@ -199,7 +200,7 @@ class ValueDecoderSpec extends Specification { ) ) ).asRight - ValueDecoder[UnstructEvent].parse(Symbol("key"), invalidPayloadUnstruct) mustEqual (Symbol("key"), "Unknown payload: iglu:invalid/schema/jsonschema/1-0-0").asLeft + ValueDecoder[UnstructEvent].parse(Symbol("key"), invalidPayloadUnstruct) mustEqual InvalidValue(Symbol("key"), invalidPayloadUnstruct, "Unknown payload: iglu:invalid/schema/jsonschema/1-0-0").asLeft } } }