Bump to JDK 11

.github/workflows/test.yml

@@ -20,10 +20,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 11
     - name: Prepare Mock server for SCE ApiRequestEnrichmentIntegrationTest (launch in background)
       run: python integration-tests/sce-api-lookup-test.py 8001 &
     - name: Prepare Postgres for SCE SqlLookupEnrichmentIntegrationTest (create entities)
@@ -55,10 +55,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 11
     - name: Compare SBT version with git tag
       run: .github/check_tag.sh ${GITHUB_REF##*/}
     - name: Docker login
@@ -81,10 +81,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 11
     - name: Compare SBT version with git tag
       run: .github/check_tag.sh ${GITHUB_REF##*/}
     - name: Deploy SCE on Bintray Maven and Maven Central

.jvmopts

@@ -8,6 +8,5 @@
 -XX:+TieredCompilation
 -XX:-UseGCOverheadLimit
 # effectively adds GC to Perm space
--XX:+UseConcMarkSweepGC
 -XX:+CMSClassUnloadingEnabled
 # must be enabled for CMSClassUnloadingEnabled to work

build.sbt

@@ -188,3 +188,5 @@ lazy val integrationTests = project
     Dependencies.Libraries.jinJava
   ))
   .dependsOn(stream % "test->test", kafka % "test->compile")
+
+Global / onChangedBuildSource := ReloadOnSourceChanges

modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala

@@ -36,7 +36,8 @@ object YauaaEnrichmentSpec {
     "event_format" -> "jsonschema",
     "event_version" -> "1-0-0",
     "event" -> "page_ping",
-    "derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-1","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces)
+    "derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-1","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces
+  )
 }
 
 class YauaaEnrichmentSpec extends PipelineSpec {

modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala

@@ -56,14 +56,14 @@ class EnrichedEventSpec extends Specification {
       testField(_.user_ipaddress = "user_ipaddress", _.user_ipaddress)
       testField(_.user_fingerprint = "user_fingerprint", _.user_fingerprint)
       testField(_.domain_userid = "domain_userid", _.domain_userid)
-      testField(_.domain_sessionidx = new JInteger(0), _.domain_sessionidx)
+      testField(_.domain_sessionidx = JInteger.valueOf(0), _.domain_sessionidx)
       testField(_.network_userid = "network_userid", _.network_userid)
       testField(_.geo_country = "geo_country", _.geo_country)
       testField(_.geo_region = "geo_region", _.geo_region)
       testField(_.geo_city = "geo_city", _.geo_city)
       testField(_.geo_zipcode = "geo_zipcode", _.geo_zipcode)
-      testField(_.geo_latitude = new JFloat(0.0), _.geo_latitude)
-      testField(_.geo_longitude = new JFloat(0.0), _.geo_longitude)
+      testField(_.geo_latitude = JFloat.valueOf(0.0f), _.geo_latitude)
+      testField(_.geo_longitude = JFloat.valueOf(0.0f), _.geo_longitude)
       testField(_.geo_region_name = "geo_region_name", _.geo_region_name)
       testField(_.ip_isp = "ip_isp", _.ip_isp)
       testField(_.ip_organization = "ip_organization", _.ip_organization)
@@ -74,13 +74,13 @@ class EnrichedEventSpec extends Specification {
       testField(_.page_referrer = "page_referrer", _.page_referrer)
       testField(_.page_urlscheme = "page_urlscheme", _.page_urlscheme)
       testField(_.page_urlhost = "page_urlhost", _.page_urlhost)
-      testField(_.page_urlport = new JInteger(0), _.page_urlport)
+      testField(_.page_urlport = JInteger.valueOf(0), _.page_urlport)
       testField(_.page_urlpath = "page_urlpath", _.page_urlpath)
       testField(_.page_urlquery = "page_urlquery", _.page_urlquery)
       testField(_.page_urlfragment = "page_urlfragment", _.page_urlfragment)
       testField(_.refr_urlscheme = "refr_urlscheme", _.refr_urlscheme)
       testField(_.refr_urlhost = "refr_urlhost", _.refr_urlhost)
-      testField(_.refr_urlport = new JInteger(0), _.refr_urlport)
+      testField(_.refr_urlport = JInteger.valueOf(0), _.refr_urlport)
       testField(_.refr_urlpath = "refr_urlpath", _.refr_urlpath)
       testField(_.refr_urlquery = "refr_urlquery", _.refr_urlquery)
       testField(_.refr_urlfragment = "refr_urlfragment", _.refr_urlfragment)
@@ -112,42 +112,42 @@ class EnrichedEventSpec extends Specification {
       testField(_.ti_name = "ti_name", _.ti_name)
       testField(_.ti_category = "ti_category", _.ti_category)
       testField(_.ti_price = "ti_price", _.ti_price)
-      testField(_.ti_quantity = new JInteger(0), _.ti_quantity)
-      testField(_.pp_xoffset_min = new JInteger(0), _.pp_xoffset_min)
-      testField(_.pp_xoffset_max = new JInteger(0), _.pp_xoffset_max)
-      testField(_.pp_yoffset_min = new JInteger(0), _.pp_yoffset_min)
-      testField(_.pp_yoffset_max = new JInteger(0), _.pp_yoffset_max)
+      testField(_.ti_quantity = JInteger.valueOf(0), _.ti_quantity)
+      testField(_.pp_xoffset_min = JInteger.valueOf(0), _.pp_xoffset_min)
+      testField(_.pp_xoffset_max = JInteger.valueOf(0), _.pp_xoffset_max)
+      testField(_.pp_yoffset_min = JInteger.valueOf(0), _.pp_yoffset_min)
+      testField(_.pp_yoffset_max = JInteger.valueOf(0), _.pp_yoffset_max)
       testField(_.useragent = "useragent", _.useragent)
       testField(_.br_name = "br_name", _.br_name)
       testField(_.br_family = "br_family", _.br_family)
       testField(_.br_version = "br_version", _.br_version)
       testField(_.br_type = "br_type", _.br_type)
       testField(_.br_renderengine = "br_renderengine", _.br_renderengine)
       testField(_.br_lang = "br_lang", _.br_lang)
-      testField(_.br_features_pdf = new JByte(Byte.MinValue), _.br_features_pdf)
-      testField(_.br_features_flash = new JByte(Byte.MinValue), _.br_features_flash)
-      testField(_.br_features_java = new JByte(Byte.MinValue), _.br_features_java)
-      testField(_.br_features_director = new JByte(Byte.MinValue), _.br_features_director)
-      testField(_.br_features_quicktime = new JByte(Byte.MinValue), _.br_features_quicktime)
-      testField(_.br_features_realplayer = new JByte(Byte.MinValue), _.br_features_realplayer)
-      testField(_.br_features_windowsmedia = new JByte(Byte.MinValue), _.br_features_windowsmedia)
-      testField(_.br_features_gears = new JByte(Byte.MinValue), _.br_features_gears)
-      testField(_.br_features_silverlight = new JByte(Byte.MinValue), _.br_features_silverlight)
-      testField(_.br_cookies = new JByte(Byte.MinValue), _.br_cookies)
+      testField(_.br_features_pdf = JByte.valueOf(Byte.MinValue), _.br_features_pdf)
+      testField(_.br_features_flash = JByte.valueOf(Byte.MinValue), _.br_features_flash)
+      testField(_.br_features_java = JByte.valueOf(Byte.MinValue), _.br_features_java)
+      testField(_.br_features_director = JByte.valueOf(Byte.MinValue), _.br_features_director)
+      testField(_.br_features_quicktime = JByte.valueOf(Byte.MinValue), _.br_features_quicktime)
+      testField(_.br_features_realplayer = JByte.valueOf(Byte.MinValue), _.br_features_realplayer)
+      testField(_.br_features_windowsmedia = JByte.valueOf(Byte.MinValue), _.br_features_windowsmedia)
+      testField(_.br_features_gears = JByte.valueOf(Byte.MinValue), _.br_features_gears)
+      testField(_.br_features_silverlight = JByte.valueOf(Byte.MinValue), _.br_features_silverlight)
+      testField(_.br_cookies = JByte.valueOf(Byte.MinValue), _.br_cookies)
       testField(_.br_colordepth = "br_colordepth", _.br_colordepth)
-      testField(_.br_viewwidth = new JInteger(0), _.br_viewwidth)
-      testField(_.br_viewheight = new JInteger(0), _.br_viewheight)
+      testField(_.br_viewwidth = JInteger.valueOf(0), _.br_viewwidth)
+      testField(_.br_viewheight = JInteger.valueOf(0), _.br_viewheight)
       testField(_.os_name = "os_name", _.os_name)
       testField(_.os_family = "os_family", _.os_family)
       testField(_.os_manufacturer = "os_manufacturer", _.os_manufacturer)
       testField(_.os_timezone = "os_timezone", _.os_timezone)
       testField(_.dvce_type = "dvce_type", _.dvce_type)
-      testField(_.dvce_ismobile = new JByte(Byte.MinValue), _.dvce_ismobile)
-      testField(_.dvce_screenwidth = new JInteger(0), _.dvce_screenwidth)
-      testField(_.dvce_screenheight = new JInteger(0), _.dvce_screenheight)
+      testField(_.dvce_ismobile = JByte.valueOf(Byte.MinValue), _.dvce_ismobile)
+      testField(_.dvce_screenwidth = JInteger.valueOf(0), _.dvce_screenwidth)
+      testField(_.dvce_screenheight = JInteger.valueOf(0), _.dvce_screenheight)
       testField(_.doc_charset = "doc_charset", _.doc_charset)
-      testField(_.doc_width = new JInteger(0), _.doc_width)
-      testField(_.doc_height = new JInteger(0), _.doc_height)
+      testField(_.doc_width = JInteger.valueOf(0), _.doc_width)
+      testField(_.doc_height = JInteger.valueOf(0), _.doc_height)
       testField(_.tr_currency = "tr_currency", _.tr_currency)
       testField(_.tr_total_base = "tr_total_base", _.tr_total_base)
       testField(_.tr_tax_base = "tr_tax_base", _.tr_tax_base)
@@ -171,7 +171,6 @@ class EnrichedEventSpec extends Specification {
       testField(_.event_version = "event_version", _.event_version)
       testField(_.event_fingerprint = "event_fingerprint", _.event_fingerprint)
       testField(_.true_tstamp = "true_tstamp", _.true_tstamp)
-
     }
   }
 

modules/integration-tests/src/test/scala/com.snowplowanalytics.snowplow.enrich.stream/KafkaTestUtils.scala

@@ -106,7 +106,7 @@ class KafkaTestUtils {
   @scala.annotation.varargs
   def createTopics(topics: String*): Unit =
     for (topic <- topics) {
-      adminClient.createTopics(List(new NewTopic(topic, 1, 1)).asJava)
+      adminClient.createTopics(List(new NewTopic(topic, 1, 1.toShort)).asJava)
       Thread.sleep(1000)
       topicCountMap = topicCountMap + (topic -> 1)
     }

modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sinks/KafkaSink.scala

@@ -22,8 +22,6 @@ package sinks
 
 import java.util.Properties
 
-import scala.collection.JavaConverters._
-
 import cats.syntax.either._
 import org.apache.kafka.clients.producer._
 
@@ -41,7 +39,7 @@ object KafkaSink {
    */
   private def createProducer(kafkaConfig: Kafka, bufferConfig: BufferConfig): KafkaProducer[String, String] = {
     val properties = createProperties(kafkaConfig, bufferConfig)
-    properties.putAll(kafkaConfig.producerConf.getOrElse(Map()).asJava)
+    kafkaConfig.producerConf.getOrElse(Map()).foreach { case (k, v) => properties.setProperty(k, v) }
     new KafkaProducer[String, String](properties)
   }
 

modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sources/KafkaSource.scala

@@ -133,7 +133,7 @@ class KafkaSource private (
 
   private def createConsumer(brokers: String, groupId: String): KafkaConsumer[String, Array[Byte]] = {
     val properties = createProperties(brokers, groupId)
-    properties.putAll(kafkaConfig.consumerConf.getOrElse(Map()).asJava)
+    kafkaConfig.consumerConf.getOrElse(Map()).foreach { case (k, v) => properties.setProperty(k, v) }
     new KafkaConsumer[String, Array[Byte]](properties)
   }
 

project/BuildSettings.scala

@@ -37,8 +37,9 @@ object BuildSettings {
     organization          :=  "com.snowplowanalytics",
     scalaVersion          :=  "2.12.11",
     version               :=  "1.3.2",
-    javacOptions          :=  Seq("-source", "1.8", "-target", "1.8"),
-    resolvers             ++= Dependencies.resolutionRepos
+    javacOptions          :=  Seq("-source", "11", "-target", "11"),
+    resolvers             ++= Dependencies.resolutionRepos,
+    useCoursier           := false
   )
 
   /** Custom sbt-buildinfo replacement, used by SCE only */
@@ -120,7 +121,7 @@ object BuildSettings {
   /** Docker settings, used by SE */
   lazy val dockerSettings = Seq(
     maintainer in Docker := "Snowplow Analytics Ltd. <support@snowplowanalytics.com>",
-    dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0",
+    dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.2.1",
     daemonUser in Docker := "snowplow",
     dockerUpdateLatest := true,
     dockerVersion := Some(DockerVersion(18, 9, 0, Some("ce"))),

project/Dependencies.scala

@@ -61,7 +61,7 @@ object Dependencies {
     val awsSdk           = "1.11.728"
     val gcpSdk           = "1.106.0"
     val kinesisClient    = "1.13.3"
-    val kafka            = "2.2.1"
+    val kafka            = "2.6.0"
     val nsqClient        = "1.2.0"
     val jackson          = "2.10.3"
     val config           = "1.3.4"