From 4f7762f718ef43a0ab5802f2d11576203774da4c Mon Sep 17 00:00:00 2001 From: Oguzhan Unlu Date: Mon, 21 Sep 2020 14:15:53 +0300 Subject: [PATCH] Bump to JDK 11 --- .github/workflows/test.yml | 12 ++-- .jvmopts | 1 - build.sbt | 2 + .../enrichments/YauaaEnrichmentSpec.scala | 3 +- .../outputs/EnrichedEventSpec.scala | 55 +++++++++---------- .../KafkaTestUtils.scala | 2 +- .../sinks/KafkaSink.scala | 4 +- .../sources/KafkaSource.scala | 2 +- project/BuildSettings.scala | 7 ++- project/Dependencies.scala | 2 +- 10 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 616b62a30..ab63058e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,10 +20,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Prepare Mock server for SCE ApiRequestEnrichmentIntegrationTest (launch in background) run: python integration-tests/sce-api-lookup-test.py 8001 & - name: Prepare Postgres for SCE SqlLookupEnrichmentIntegrationTest (create entities) @@ -55,10 +55,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Compare SBT version with git tag run: .github/check_tag.sh ${GITHUB_REF##*/} - name: Docker login @@ -81,10 +81,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Compare SBT version with git tag run: .github/check_tag.sh ${GITHUB_REF##*/} - name: Deploy SCE on Bintray Maven and Maven Central diff --git a/.jvmopts b/.jvmopts index ef0f90285..00c1707db 100644 --- a/.jvmopts +++ b/.jvmopts @@ -8,6 +8,5 @@ -XX:+TieredCompilation -XX:-UseGCOverheadLimit # effectively adds GC to Perm space --XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled # must be enabled for CMSClassUnloadingEnabled to work diff --git a/build.sbt b/build.sbt index 09db15707..629ab51f4 100644 --- a/build.sbt +++ b/build.sbt @@ -188,3 +188,5 @@ lazy val integrationTests = project Dependencies.Libraries.jinJava )) .dependsOn(stream % "test->test", kafka % "test->compile") + +Global / onChangedBuildSource := ReloadOnSourceChanges diff --git a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala index 1610c046b..ac0288786 100644 --- a/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala +++ b/modules/beam/src/test/scala/com.snowplowanalytics.snowplow.enrich.beam/enrichments/YauaaEnrichmentSpec.scala @@ -36,7 +36,8 @@ object YauaaEnrichmentSpec { "event_format" -> "jsonschema", "event_version" -> "1-0-0", "event" -> "page_ping", - "derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-1","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces) + "derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-1","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces + ) } class YauaaEnrichmentSpec extends PipelineSpec { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala index 5c118448c..46dae268f 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala @@ -56,14 +56,14 @@ class EnrichedEventSpec extends Specification { testField(_.user_ipaddress = "user_ipaddress", _.user_ipaddress) testField(_.user_fingerprint = "user_fingerprint", _.user_fingerprint) testField(_.domain_userid = "domain_userid", _.domain_userid) - testField(_.domain_sessionidx = new JInteger(0), _.domain_sessionidx) + testField(_.domain_sessionidx = JInteger.valueOf(0), _.domain_sessionidx) testField(_.network_userid = "network_userid", _.network_userid) testField(_.geo_country = "geo_country", _.geo_country) testField(_.geo_region = "geo_region", _.geo_region) testField(_.geo_city = "geo_city", _.geo_city) testField(_.geo_zipcode = "geo_zipcode", _.geo_zipcode) - testField(_.geo_latitude = new JFloat(0.0), _.geo_latitude) - testField(_.geo_longitude = new JFloat(0.0), _.geo_longitude) + testField(_.geo_latitude = JFloat.valueOf(0.0f), _.geo_latitude) + testField(_.geo_longitude = JFloat.valueOf(0.0f), _.geo_longitude) testField(_.geo_region_name = "geo_region_name", _.geo_region_name) testField(_.ip_isp = "ip_isp", _.ip_isp) testField(_.ip_organization = "ip_organization", _.ip_organization) @@ -74,13 +74,13 @@ class EnrichedEventSpec extends Specification { testField(_.page_referrer = "page_referrer", _.page_referrer) testField(_.page_urlscheme = "page_urlscheme", _.page_urlscheme) testField(_.page_urlhost = "page_urlhost", _.page_urlhost) - testField(_.page_urlport = new JInteger(0), _.page_urlport) + testField(_.page_urlport = JInteger.valueOf(0), _.page_urlport) testField(_.page_urlpath = "page_urlpath", _.page_urlpath) testField(_.page_urlquery = "page_urlquery", _.page_urlquery) testField(_.page_urlfragment = "page_urlfragment", _.page_urlfragment) testField(_.refr_urlscheme = "refr_urlscheme", _.refr_urlscheme) testField(_.refr_urlhost = "refr_urlhost", _.refr_urlhost) - testField(_.refr_urlport = new JInteger(0), _.refr_urlport) + testField(_.refr_urlport = JInteger.valueOf(0), _.refr_urlport) testField(_.refr_urlpath = "refr_urlpath", _.refr_urlpath) testField(_.refr_urlquery = "refr_urlquery", _.refr_urlquery) testField(_.refr_urlfragment = "refr_urlfragment", _.refr_urlfragment) @@ -112,11 +112,11 @@ class EnrichedEventSpec extends Specification { testField(_.ti_name = "ti_name", _.ti_name) testField(_.ti_category = "ti_category", _.ti_category) testField(_.ti_price = "ti_price", _.ti_price) - testField(_.ti_quantity = new JInteger(0), _.ti_quantity) - testField(_.pp_xoffset_min = new JInteger(0), _.pp_xoffset_min) - testField(_.pp_xoffset_max = new JInteger(0), _.pp_xoffset_max) - testField(_.pp_yoffset_min = new JInteger(0), _.pp_yoffset_min) - testField(_.pp_yoffset_max = new JInteger(0), _.pp_yoffset_max) + testField(_.ti_quantity = JInteger.valueOf(0), _.ti_quantity) + testField(_.pp_xoffset_min = JInteger.valueOf(0), _.pp_xoffset_min) + testField(_.pp_xoffset_max = JInteger.valueOf(0), _.pp_xoffset_max) + testField(_.pp_yoffset_min = JInteger.valueOf(0), _.pp_yoffset_min) + testField(_.pp_yoffset_max = JInteger.valueOf(0), _.pp_yoffset_max) testField(_.useragent = "useragent", _.useragent) testField(_.br_name = "br_name", _.br_name) testField(_.br_family = "br_family", _.br_family) @@ -124,30 +124,30 @@ class EnrichedEventSpec extends Specification { testField(_.br_type = "br_type", _.br_type) testField(_.br_renderengine = "br_renderengine", _.br_renderengine) testField(_.br_lang = "br_lang", _.br_lang) - testField(_.br_features_pdf = new JByte(Byte.MinValue), _.br_features_pdf) - testField(_.br_features_flash = new JByte(Byte.MinValue), _.br_features_flash) - testField(_.br_features_java = new JByte(Byte.MinValue), _.br_features_java) - testField(_.br_features_director = new JByte(Byte.MinValue), _.br_features_director) - testField(_.br_features_quicktime = new JByte(Byte.MinValue), _.br_features_quicktime) - testField(_.br_features_realplayer = new JByte(Byte.MinValue), _.br_features_realplayer) - testField(_.br_features_windowsmedia = new JByte(Byte.MinValue), _.br_features_windowsmedia) - testField(_.br_features_gears = new JByte(Byte.MinValue), _.br_features_gears) - testField(_.br_features_silverlight = new JByte(Byte.MinValue), _.br_features_silverlight) - testField(_.br_cookies = new JByte(Byte.MinValue), _.br_cookies) + testField(_.br_features_pdf = JByte.valueOf(Byte.MinValue), _.br_features_pdf) + testField(_.br_features_flash = JByte.valueOf(Byte.MinValue), _.br_features_flash) + testField(_.br_features_java = JByte.valueOf(Byte.MinValue), _.br_features_java) + testField(_.br_features_director = JByte.valueOf(Byte.MinValue), _.br_features_director) + testField(_.br_features_quicktime = JByte.valueOf(Byte.MinValue), _.br_features_quicktime) + testField(_.br_features_realplayer = JByte.valueOf(Byte.MinValue), _.br_features_realplayer) + testField(_.br_features_windowsmedia = JByte.valueOf(Byte.MinValue), _.br_features_windowsmedia) + testField(_.br_features_gears = JByte.valueOf(Byte.MinValue), _.br_features_gears) + testField(_.br_features_silverlight = JByte.valueOf(Byte.MinValue), _.br_features_silverlight) + testField(_.br_cookies = JByte.valueOf(Byte.MinValue), _.br_cookies) testField(_.br_colordepth = "br_colordepth", _.br_colordepth) - testField(_.br_viewwidth = new JInteger(0), _.br_viewwidth) - testField(_.br_viewheight = new JInteger(0), _.br_viewheight) + testField(_.br_viewwidth = JInteger.valueOf(0), _.br_viewwidth) + testField(_.br_viewheight = JInteger.valueOf(0), _.br_viewheight) testField(_.os_name = "os_name", _.os_name) testField(_.os_family = "os_family", _.os_family) testField(_.os_manufacturer = "os_manufacturer", _.os_manufacturer) testField(_.os_timezone = "os_timezone", _.os_timezone) testField(_.dvce_type = "dvce_type", _.dvce_type) - testField(_.dvce_ismobile = new JByte(Byte.MinValue), _.dvce_ismobile) - testField(_.dvce_screenwidth = new JInteger(0), _.dvce_screenwidth) - testField(_.dvce_screenheight = new JInteger(0), _.dvce_screenheight) + testField(_.dvce_ismobile = JByte.valueOf(Byte.MinValue), _.dvce_ismobile) + testField(_.dvce_screenwidth = JInteger.valueOf(0), _.dvce_screenwidth) + testField(_.dvce_screenheight = JInteger.valueOf(0), _.dvce_screenheight) testField(_.doc_charset = "doc_charset", _.doc_charset) - testField(_.doc_width = new JInteger(0), _.doc_width) - testField(_.doc_height = new JInteger(0), _.doc_height) + testField(_.doc_width = JInteger.valueOf(0), _.doc_width) + testField(_.doc_height = JInteger.valueOf(0), _.doc_height) testField(_.tr_currency = "tr_currency", _.tr_currency) testField(_.tr_total_base = "tr_total_base", _.tr_total_base) testField(_.tr_tax_base = "tr_tax_base", _.tr_tax_base) @@ -171,7 +171,6 @@ class EnrichedEventSpec extends Specification { testField(_.event_version = "event_version", _.event_version) testField(_.event_fingerprint = "event_fingerprint", _.event_fingerprint) testField(_.true_tstamp = "true_tstamp", _.true_tstamp) - } } diff --git a/modules/integration-tests/src/test/scala/com.snowplowanalytics.snowplow.enrich.stream/KafkaTestUtils.scala b/modules/integration-tests/src/test/scala/com.snowplowanalytics.snowplow.enrich.stream/KafkaTestUtils.scala index 6ae18b881..27c279670 100644 --- a/modules/integration-tests/src/test/scala/com.snowplowanalytics.snowplow.enrich.stream/KafkaTestUtils.scala +++ b/modules/integration-tests/src/test/scala/com.snowplowanalytics.snowplow.enrich.stream/KafkaTestUtils.scala @@ -106,7 +106,7 @@ class KafkaTestUtils { @scala.annotation.varargs def createTopics(topics: String*): Unit = for (topic <- topics) { - adminClient.createTopics(List(new NewTopic(topic, 1, 1)).asJava) + adminClient.createTopics(List(new NewTopic(topic, 1, 1.toShort)).asJava) Thread.sleep(1000) topicCountMap = topicCountMap + (topic -> 1) } diff --git a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sinks/KafkaSink.scala b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sinks/KafkaSink.scala index 889911594..b3175b060 100644 --- a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sinks/KafkaSink.scala +++ b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sinks/KafkaSink.scala @@ -22,8 +22,6 @@ package sinks import java.util.Properties -import scala.collection.JavaConverters._ - import cats.syntax.either._ import org.apache.kafka.clients.producer._ @@ -41,7 +39,7 @@ object KafkaSink { */ private def createProducer(kafkaConfig: Kafka, bufferConfig: BufferConfig): KafkaProducer[String, String] = { val properties = createProperties(kafkaConfig, bufferConfig) - properties.putAll(kafkaConfig.producerConf.getOrElse(Map()).asJava) + kafkaConfig.producerConf.getOrElse(Map()).foreach { case (k, v) => properties.setProperty(k, v) } new KafkaProducer[String, String](properties) } diff --git a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sources/KafkaSource.scala b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sources/KafkaSource.scala index 1f6fe48d7..214754dfe 100644 --- a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sources/KafkaSource.scala +++ b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.stream/sources/KafkaSource.scala @@ -133,7 +133,7 @@ class KafkaSource private ( private def createConsumer(brokers: String, groupId: String): KafkaConsumer[String, Array[Byte]] = { val properties = createProperties(brokers, groupId) - properties.putAll(kafkaConfig.consumerConf.getOrElse(Map()).asJava) + kafkaConfig.consumerConf.getOrElse(Map()).foreach { case (k, v) => properties.setProperty(k, v) } new KafkaConsumer[String, Array[Byte]](properties) } diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 21f92c32b..0f9fd3ea3 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -37,8 +37,9 @@ object BuildSettings { organization := "com.snowplowanalytics", scalaVersion := "2.12.11", version := "1.3.2", - javacOptions := Seq("-source", "1.8", "-target", "1.8"), - resolvers ++= Dependencies.resolutionRepos + javacOptions := Seq("-source", "11", "-target", "11"), + resolvers ++= Dependencies.resolutionRepos, + useCoursier := false ) /** Custom sbt-buildinfo replacement, used by SCE only */ @@ -120,7 +121,7 @@ object BuildSettings { /** Docker settings, used by SE */ lazy val dockerSettings = Seq( maintainer in Docker := "Snowplow Analytics Ltd. ", - dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0", + dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.2.1", daemonUser in Docker := "snowplow", dockerUpdateLatest := true, dockerVersion := Some(DockerVersion(18, 9, 0, Some("ce"))), diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a45685850..c851f6932 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -61,7 +61,7 @@ object Dependencies { val awsSdk = "1.11.728" val gcpSdk = "1.106.0" val kinesisClient = "1.13.3" - val kafka = "2.2.1" + val kafka = "2.6.0" val nsqClient = "1.2.0" val jackson = "2.10.3" val config = "1.3.4"