diff --git a/.travis.yml b/.travis.yml index 0d62f69..2748c1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ dist: trusty language: scala scala: - - 2.11.12 - - 2.12.8 + - 2.12.11 + - 2.13.2 jdk: - oraclejdk8 script: diff --git a/CHANGELOG b/CHANGELOG index 27e8f27..2dba2b7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +Version 2.0.0 (2020-06-09) +-------------------------- +Remove run manifest (#102) +Add Scala 2.13 support (#101) +Bump sbt-scoverage to 1.6.1 (#104) +Bump Scala to 2.12.11 (#100) +Bump sbt to 1.3.10 (#99) +Bump iglu-core-circe to 1.0.0 (#98) + Version 1.0.0 (2019-11-06) -------------------------- Make parsing errors type-safe (#75) @@ -49,7 +58,7 @@ Add Scala 2.12 support (#41) Bump json4s to 3.2.11 (#46) Bump aws-java-sdk to 1.11.289 (#48) Bump Scala 2.11 to 2.11.12 (#47) -Bump SBT to 1.1.1 (close #49) +Bump SBT to 1.1.1 (#49) Extend copyright notice to 2018 (#51) Change se_value to Double (#52) diff --git a/README.md b/README.md index cbce12f..9a8d734 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ limitations under the License. [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat [license]: http://www.apache.org/licenses/LICENSE-2.0 -[release-image]: http://img.shields.io/badge/release-1.0.0-blue.svg?style=flat +[release-image]: http://img.shields.io/badge/release-2.0.0-blue.svg?style=flat [releases]: https://github.com/snowplow/snowplow-scala-analytics-sdk/releases [setup-guide]: https://github.com/snowplow/snowplow/wiki/Scala-Analytics-SDK-setup diff --git a/build.sbt b/build.sbt index 0c07093..b3c2a2c 100644 --- a/build.sbt +++ b/build.sbt @@ -15,10 +15,10 @@ lazy val root = project.in(file(".")) .settings(Seq[Setting[_]]( name := "snowplow-scala-analytics-sdk", organization := "com.snowplowanalytics", - version := "1.0.0", + version := "2.0.0", description := "Scala analytics SDK for Snowplow", - scalaVersion := "2.12.8", - crossScalaVersions := Seq("2.11.12", "2.12.8") + scalaVersion := "2.13.2", + crossScalaVersions := Seq("2.12.11", "2.13.2") )) .enablePlugins(SiteScaladocPlugin) .enablePlugins(GhpagesPlugin) @@ -38,9 +38,6 @@ lazy val root = project.in(file(".")) Dependencies.cats, Dependencies.circeParser, Dependencies.circeGeneric, - Dependencies.circeJava, - Dependencies.s3, - Dependencies.dynamodb, // Scala (test only) Dependencies.specs2 ) diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 4a0af3f..f1265b0 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -42,12 +42,8 @@ object BuildSettings { "-feature", "-unchecked", "-Ywarn-dead-code", - "-Ywarn-inaccessible", - "-Ywarn-nullary-override", - "-Ywarn-nullary-unit", "-Ywarn-numeric-widen", - "-Ywarn-value-discard", - "-Ypartial-unification" + "-Ywarn-value-discard" ) ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 365f4cb..7d19fc0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -15,21 +15,17 @@ import sbt._ object Dependencies { object V { - val igluCore = "0.5.1" - val cats = "1.6.0" - val circe = "0.11.1" - val aws = "1.11.490" + val igluCore = "1.0.0" + val cats = "2.1.1" + val circe = "0.13.0" // Scala (test only) - val specs2 = "4.4.1" + val specs2 = "4.8.0" } val igluCore = "com.snowplowanalytics" %% "iglu-core-circe" % V.igluCore val cats = "org.typelevel" %% "cats-core" % V.cats val circeParser = "io.circe" %% "circe-parser" % V.circe val circeGeneric = "io.circe" %% "circe-generic" % V.circe - val circeJava = "io.circe" %% "circe-java8" % V.circe - val s3 = "com.amazonaws" % "aws-java-sdk-s3" % V.aws - val dynamodb = "com.amazonaws" % "aws-java-sdk-dynamodb" % V.aws // Scala (test only) - val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % "test" + val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % Test } diff --git a/project/build.properties b/project/build.properties index c0bab04..797e7cc 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 +sbt.version=1.3.10 diff --git a/project/plugins.sbt b/project/plugins.sbt index 3db8a8c..e2d1eb6 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,6 @@ addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.3") addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.5.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.0") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.3") diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/Event.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/Event.scala index a9f722e..71c3634 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/Event.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/Event.scala @@ -17,15 +17,14 @@ import java.time.Instant import java.util.UUID // circe -import io.circe.{Encoder, Json, JsonObject, ObjectEncoder, Decoder} +import io.circe.{Encoder, Json, JsonObject, Decoder} import io.circe.Json.JString import io.circe.generic.semiauto._ import io.circe.syntax._ -import io.circe.java8.time._ // iglu import com.snowplowanalytics.iglu.core.SelfDescribingData -import com.snowplowanalytics.iglu.core.circe.instances._ +import com.snowplowanalytics.iglu.core.circe.implicits._ // This library import com.snowplowanalytics.snowplow.analytics.scalasdk.decode.{Parser, DecodeResult} @@ -239,7 +238,7 @@ object Event { /** * Automatically derived Circe encoder */ - implicit val jsonEncoder: ObjectEncoder[Event] = deriveEncoder[Event] + implicit val jsonEncoder: Encoder.AsObject[Event] = deriveEncoder[Event] implicit def eventDecoder: Decoder[Event] = deriveDecoder[Event] @@ -270,4 +269,4 @@ object Event { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, Contexts(Nil), None, None, None, None, None, None, None, None) -} \ No newline at end of file +} diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/RunManifests.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/RunManifests.scala deleted file mode 100644 index a43f80b..0000000 --- a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/RunManifests.scala +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.snowplow.analytics.scalasdk - -// Scala -import scala.collection.convert.decorateAsJava._ -import scala.collection.convert.decorateAsScala._ - -// AWS DynamoDB -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB -import com.amazonaws.services.dynamodbv2.model._ -import com.amazonaws.services.dynamodbv2.document.Table - -// AWS S3 -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.{ ListObjectsV2Request, ListObjectsV2Result } - -/** - * Wrapper class for run manifests table - * Can be used instead of functions from `RunManifests` module - * - * @param dynamodb AWS DynamoDB client - * @param tableName existing DynamoDB table name with run manifests - */ -@deprecated("In favor of https://github.com/snowplow-incubator/snowplow-processing-manifest/", "snowplow-scala-analytics-sdk 1.0.0") -class RunManifests(dynamodb: AmazonDynamoDB, tableName: String) { - /** - * Creates DynamoDB table with all necessary settings - * Should be called once, will throw exception otherwise - */ - def create(): Unit = - RunManifests.createManifestsTable(dynamodb, tableName) - - /** - * Check whether run manifests table contains particular run id - * - * @param runId run id to check - * @return true if run id is in table - */ - def contains(runId: String): Boolean = - RunManifests.isInManifests(dynamodb, tableName, runId) - - /** - * Add run id to manifests table - * - * @param runId run id to add - */ - def add(runId: String): Unit = - RunManifests.addToManifests(dynamodb, tableName, runId) -} - - -/** - * Module with primary run-manifests functions, without applied client - */ -@deprecated("In favor of https://github.com/snowplow-incubator/snowplow-processing-manifest/", "snowplow-scala-analytics-sdk 1.0.0") -object RunManifests { - - /** - * S3 protocol implementations that can be used for `listRunIds` `fullPath` argument - */ - val SupportedPrefixes = Set("s3://", "s3n://", "s3a://") - - /** - * Standard attribute name containing run id - */ - val DynamoDbRunIdAttribute = "RunId" - - /** - * `RunManifests` short-hand constructor - */ - def apply(dynamodb: AmazonDynamoDB, tableName: String): RunManifests = - new RunManifests(dynamodb, tableName) - - /** - * List *all* (regardless `MaxKeys`) run ids in specified S3 path - * (such as enriched-archive) - * - * @param s3 AWS S3 client - * @param fullPath full S3 path (including bucket and prefix) to folder with run ids - * @return list of prefixes (without S3 bucket) - * such as `storage/enriched/good/run=2017-02-21-11-40-15` - */ - def listRunIds(s3: AmazonS3, fullPath: String): List[String] = { - // Initialize mutable buffer - val buffer = collection.mutable.ListBuffer.empty[String] - var result: ListObjectsV2Result = null - - val (bucket, prefix) = splitFullPath(fullPath) match { - case Right((b, p)) => (b, p) - case Left(error) => throw new RuntimeException(error) - } - - val glacierified: String => Boolean = - isGlacierified(s3, bucket, _) - - val req = new ListObjectsV2Request() - .withBucketName(bucket) - .withDelimiter("/") - .withPrefix(prefix.orNull) - - do { - result = s3.listObjectsV2(req) - val objects = result.getCommonPrefixes.asScala.toList.filterNot(glacierified) - buffer ++= objects - - req.setContinuationToken(result.getNextContinuationToken) - } while(result.isTruncated) - - buffer.toList - } - - - /** - * Check if specified prefix (directory) contains objects archived to AWS Glacier - * - * @param s3 AWS S3 client - * @param bucket AWS S3 bucket (without prefix and trailing slash) - * @param prefix full prefix with trailing slash - * @return true if any of first 3 objects has GLACIER storage class - */ - private def isGlacierified(s3: AmazonS3, bucket: String, prefix: String): Boolean = { - val req = new ListObjectsV2Request() - .withBucketName(bucket) - .withPrefix(prefix) - .withMaxKeys(3) // Use 3 to not accidentally fetch _SUCCESS or ghost files - - val classes = s3.listObjectsV2(req).getObjectSummaries.asScala.map(_.getStorageClass) - classes.contains("GLACIER") - } - - /** - * Creates DynamoDB table with all necessary settings - * Should be called once, will throw exception otherwise - * - * @param dynamodb AWS DynamoDB client - * @param tableName existing DynamoDB table name with run manifests - */ - def createManifestsTable(dynamodb: AmazonDynamoDB, tableName: String): Unit = { - val runIdAttribute = new AttributeDefinition(RunManifests.DynamoDbRunIdAttribute, ScalarAttributeType.S) - val manifestsSchema = new KeySchemaElement(RunManifests.DynamoDbRunIdAttribute, KeyType.HASH) - val manifestsThroughput = new ProvisionedThroughput(5L, 5L) - val req = new CreateTableRequest() - .withTableName(tableName) - .withAttributeDefinitions(runIdAttribute) - .withKeySchema(manifestsSchema) - .withProvisionedThroughput(manifestsThroughput) - - try { - dynamodb.createTable(req) - } catch { - case _: ResourceInUseException => () - } - RunManifests.waitForActive(dynamodb, tableName) - } - - /** - * Check whether run manifests table contains particular run id - * - * @param dynamodb AWS DynamoDB client - * @param tableName existing DynamoDB table name with run manifests - * @param runId run id to check - * @return true if run id is in table - */ - def isInManifests(dynamodb: AmazonDynamoDB, tableName: String, runId: String): Boolean = { - val key = Map(RunManifests.DynamoDbRunIdAttribute -> new AttributeValue(runId)).asJava - val request = new GetItemRequest() - .withTableName(tableName) - .withKey(key) - .withAttributesToGet(RunManifests.DynamoDbRunIdAttribute) - Option(dynamodb.getItem(request).getItem).isDefined - } - - /** - * Add run id to manifests table - * - * @param dynamodb AWS DynamoDB client - * @param tableName existing DynamoDB table name with run manifests - * @param runId run id to add - */ - def addToManifests(dynamodb: AmazonDynamoDB, tableName: String, runId: String): Unit = { - val request = new PutItemRequest() - .withTableName(tableName) - .withItem(Map(RunManifests.DynamoDbRunIdAttribute -> new AttributeValue(runId)).asJava) - dynamodb.putItem(request) - } - - /** - * Wait until table is in `ACTIVE` state - * - * @param client AWS DynamoDB client - * @param name DynamoDB table name - */ - private def waitForActive(client: AmazonDynamoDB, name: String): Unit = - new Table(client, name).waitForActive() - - /** - * Split full S3 path with bucket and - * - * @param fullS3Path full S3 path with protocol, bucket name and optionally prefix - * @return pair of bucket name (without "s3://) and prefix if it was specified - * (otherwise we're looking at root of bucket) - */ - private[scalasdk] def splitFullPath(fullS3Path: String): Either[String, (String, Option[String])] = { - stripPrefix(fullS3Path) match { - case Right(path) => - val parts = path.split("/").toList - parts match { - case bucket :: Nil => Right((bucket, None)) - case bucket :: prefix => Right((bucket, Some(prefix.mkString("/") + "/"))) - case _ => Left("Cannot split path") // Cannot happen - } - case Left(error) => Left(error) - } - } - - /** - * Remove one of allowed prefixed (s3://, s3n:// etc) from S3 path - * - * @param path full S3 path with prefix - * @return right S3 path without prefix or error - */ - private[scalasdk] def stripPrefix(path: String): Either[String, String] = { - val error: Either[String, String] = - Left(s"S3 path [$path] doesn't start with one of possible prefixes: [${SupportedPrefixes.mkString(", ")}]") - - SupportedPrefixes.foldLeft(error) { (result, prefix) => - result match { - case Left(_) if path.startsWith(prefix) => Right(path.stripPrefix(prefix)) - case other => other - } - } - } -} diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala index b28f8b7..2241655 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala @@ -25,7 +25,7 @@ import cats.syntax.show._ // iglu import com.snowplowanalytics.iglu.core.SelfDescribingData -import com.snowplowanalytics.iglu.core.circe.instances._ +import com.snowplowanalytics.iglu.core.circe.implicits._ // circe import io.circe.parser.{parse => parseJson} diff --git a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala index f44bd1f..e1457f9 100644 --- a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala +++ b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/EventSpec.scala @@ -717,7 +717,7 @@ class EventSpec extends Specification { "event_version": "1-0-0", "event_fingerprint": "e3dbfa9cca0412c3d4052863cefb547f", "true_tstamp": "2013-11-26T00:03:57.886Z" - }""").right.getOrElse(throw new RuntimeException("Failed to parse expected JSON")) + }""").getOrElse(throw new RuntimeException("Failed to parse expected JSON")) // JSON output must be equal to output from the old transformer. (NB: field ordering in new JSON will be randomized) eventJson mustEqual expectedJson @@ -1131,7 +1131,7 @@ class EventSpec extends Specification { "event_version": "1-0-0", "event_fingerprint": "e3dbfa9cca0412c3d4052863cefb547f", "true_tstamp": "2013-11-26T00:03:57.886Z" - }""").right.getOrElse(throw new RuntimeException("Failed to parse expected JSON")) + }""").getOrElse(throw new RuntimeException("Failed to parse expected JSON")) // JSON output must be equal to output from the old transformer. (NB: field ordering in new JSON will be randomized) eventJson mustEqual expectedJson @@ -1662,7 +1662,7 @@ class EventSpec extends Specification { "event_version": "1-0-0", "event_fingerprint": "e3dbfa9cca0412c3d4052863cefb547f", "true_tstamp": "2013-11-26T00:03:57.886Z" - }""").right.getOrElse(throw new RuntimeException("Failed to parse expected JSON")) + }""").getOrElse(throw new RuntimeException("Failed to parse expected JSON")) // JSON output must be equal to output from the old transformer. (NB: field ordering in new JSON will be randomized) eventJson mustEqual expectedJson diff --git a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/RunManifestsSpec.scala b/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/RunManifestsSpec.scala deleted file mode 100644 index 22d492d..0000000 --- a/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/RunManifestsSpec.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.snowplow.analytics.scalasdk - -// Specs2 -import org.specs2.Specification - -class RunManifestsSpec extends Specification { def is = s2""" - splitFullPath works for bucket root $e1 - splitFullPath works for bucket root with trailing slash $e2 - splitFullPath works with prefix $e3 - splitFullPath works with prefix and trailing slash $e4 - """ - - def e1 = - RunManifests.splitFullPath("s3://some-bucket") must beRight(("some-bucket", None)) - - def e2 = - RunManifests.splitFullPath("s3://some-bucket/") must beRight(("some-bucket", None)) - - def e3 = - RunManifests.splitFullPath("s3://some-bucket/some/prefix") must beRight(("some-bucket", Some("some/prefix/"))) - - def e4 = - RunManifests.splitFullPath("s3://some-bucket/prefix/") must beRight(("some-bucket", Some("prefix/"))) - -}