Release/1.4.0

.github/workflows/test.yml

@@ -20,71 +20,131 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 11
     - name: Prepare Mock server for SCE ApiRequestEnrichmentIntegrationTest (launch in background)
       run: python integration-tests/sce-api-lookup-test.py 8001 &
-    - name: Prepare Postgres for SCE SqlLookupEnrichmentIntegrationTest (create entities)
-      run: psql -h localhost -p 5432 -U enricher -d sql_enrichment_test < integration-tests/sce-sql-enrichment-test.sql
-      env:
-        PGPASSWORD: supersecret1
     - name: Prepare Mock server for BE ApiRequestEnrichmentIntegrationTest (launch in background)
       run: python integration-tests/beam-api-lookup-test.py &
-    - name: Prepare Postgres for BE SqlLookupEnrichmentIntegrationTest (create entities)
-      run: psql -h localhost -p 5432 -U enricher -d sql_enrichment_test < integration-tests/beam-sql-enrichment-test.sql
+    - name: Prepare Postgres for SCE SqlLookupEnrichmentIntegrationTest (create entities)
+      run: psql -h localhost -p 5432 -U enricher -d sql_enrichment_test < integration-tests/sce-sql-enrichment-test.sql
       env:
         PGPASSWORD: supersecret1
     - name: Run tests
-      run: sbt coverage +test coverageReport
+      run: sbt "project common" +test
       env:
         OER_KEY: ${{ secrets.OER_KEY }}
-    - name: Aggregate coverage data
+    - name: Check Scala formatting
       if: ${{ always() }}
-      run: sbt coverageAggregate
-    - name: Submit coveralls data
+      run: sbt scalafmtCheck
+    - name: Check assets can be published
       if: ${{ always() }}
-      run: sbt coveralls
-      env:
-        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+      run: sbt publishLocal
 
-  deploy:
+  deploy_stream:
     needs: test
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 11
     - name: Compare SBT version with git tag
       run: .github/check_tag.sh ${GITHUB_REF##*/}
+    - name: Test stream enrich
+      run: sbt "project stream" test
     - name: Docker login
       run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
       env:
         DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
         DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    - name: Build and publish Beam Docker images
-      run: sbt "project beam" docker:publish
     - name: Build and publish Stream Kinesis Docker image
+      if: ${{ always() }}
       run: sbt "project kinesis" docker:publish
     - name: Build and publish Stream Kafka Docker image
+      if: ${{ always() }}
       run: sbt "project kafka" docker:publish
-    - name: Build and publish Stream NSQ Docker images
+    - name: Build and publish Stream NSQ Docker image
+      if: ${{ always() }}
       run: sbt "project nsq" docker:publish
 
+  deploy_fs2:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Compare SBT version with git tag
+      run: .github/check_tag.sh ${GITHUB_REF##*/}
+    - name: Test FS2 enrich
+      run: sbt "project fs2" test
+    - name: Docker login
+      run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Build and publish Stream NH Docker image
+      run: sbt "project fs2" docker:publish
+
+  deploy_beam:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: enricher
+          POSTGRES_PASSWORD: supersecret1
+          POSTGRES_DB: sql_enrichment_test
+          POSTGRES_PORT: 5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Compare SBT version with git tag
+      run: .github/check_tag.sh ${GITHUB_REF##*/}
+    - name: Docker login
+      run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Prepare Mock server for BE ApiRequestEnrichmentIntegrationTest (launch in background)
+      run: python integration-tests/beam-api-lookup-test.py &
+    - name: Prepare Postgres for BE SqlLookupEnrichmentIntegrationTest (create entities)
+      run: psql -h localhost -p 5432 -U enricher -d sql_enrichment_test < integration-tests/beam-sql-enrichment-test.sql
+      env:
+        PGPASSWORD: supersecret1
+    - name: Test Beam enrich
+      run: sbt "project beam" test
+    - name: Build and publish Beam Docker images
+      run: sbt "project beam" docker:publish
+
   deploy_sce:
     needs: test
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 11
     - name: Compare SBT version with git tag
       run: .github/check_tag.sh ${GITHUB_REF##*/}
     - name: Deploy SCE on Bintray Maven and Maven Central
@@ -94,3 +154,52 @@ jobs:
         SONA_PASS: ${{ secrets.SONA_PASS }}
         BINTRAY_SNOWPLOW_MAVEN_USER: ${{ secrets.BINTRAY_SNOWPLOW_MAVEN_USER }}
         BINTRAY_SNOWPLOW_MAVEN_API_KEY: ${{ secrets.BINTRAY_SNOWPLOW_MAVEN_API_KEY }}
+
+  coverage:
+    needs: test
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: enricher
+          POSTGRES_PASSWORD: supersecret1
+          POSTGRES_DB: sql_enrichment_test
+          POSTGRES_PORT: 5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+
+    - name: Prepare Mock server for BE ApiRequestEnrichmentIntegrationTest (launch in background)
+      run: python integration-tests/beam-api-lookup-test.py &
+    - name: Prepare Postgres for BE SqlLookupEnrichmentIntegrationTest (create entities)
+      run: psql -h localhost -p 5432 -U enricher -d sql_enrichment_test < integration-tests/beam-sql-enrichment-test.sql
+      env:
+        PGPASSWORD: supersecret1
+    - name: Prepare Mock server for SCE ApiRequestEnrichmentIntegrationTest (launch in background)
+      run: python integration-tests/sce-api-lookup-test.py 8001 &
+    - name: Prepare Postgres for SCE SqlLookupEnrichmentIntegrationTest (create entities)
+      run: psql -h localhost -p 5432 -U enricher -d sql_enrichment_test < integration-tests/sce-sql-enrichment-test.sql
+      env:
+        PGPASSWORD: supersecret1
+
+    - name: Run tests
+      run: sbt coverage +test coverageReport
+      env:
+        OER_KEY: ${{ secrets.OER_KEY }}
+    - name: Aggregate coverage data
+      if: ${{ always() }}
+      run: sbt coverageAggregate
+    - name: Submit coveralls data
+      if: ${{ always() }}
+      run: sbt coveralls
+      env:
+        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}

.jvmopts

@@ -8,6 +8,5 @@
 -XX:+TieredCompilation
 -XX:-UseGCOverheadLimit
 # effectively adds GC to Perm space
--XX:+UseConcMarkSweepGC
 -XX:+CMSClassUnloadingEnabled
 # must be enabled for CMSClassUnloadingEnabled to work

CHANGELOG

@@ -1,3 +1,43 @@
+Version 1.4.0 (2020-10-21)
+--------------------------
+Stream FS2: add (#346)
+Stream: bump log4j-core to 2.13.3 (#368)
+Stream: bump base-debian to 0.2.1 (#359)
+Stream: remove unused Maxmind database (#352)
+Beam: bump Scio to 0.9.3 (#308)
+Beam: use test Maxmind databases (#269)
+Common: add benchmarking module (#370)
+Common: bump scala-forex to 1.0.0 (#349)
+Common: bump scala-referer-parser to 1.1.0 (#348)
+Common: bump scala-weather to 1.0.0 (#347)
+Common: bump iglu-scala-client to 1.0.2 (#52)
+Common: bump scala-maxmind-iplookups to 0.7.1 (#323)
+Common: bump snowplow-badrows to 2.1.0 (#325)
+Common: bump YAUAA to 5.19 (#314)
+Common: bump postgresql to 42.2.16 (#369)
+Common: bump jackson-databind to 2.10.5 (#367)
+Common: bump to JDK 11 (#362)
+Common: switch to HostName.asInetAddress to validate IP addresses (#355)
+Common: fix NullPointerException on serializing invalid state (#371)
+Common: fix API Request Enrichment output deserialization (#374)
+Common: fix PiiPseudonymizerEnrichment for arrays and improve unit tests coverage (#334)
+Common: fix PII enrichment adding empty objects instead of missing properties (#351)
+Common: fix PathNotFoundException in PII enrichment (#339)
+Common: fix pattern matching against null in ScrambleMapFunction (#338)
+Common: fix flaky ThriftLoader test (#306)
+Common: handle empty query string parameters in adapters (#341)
+Common: make assets publishing independent of each other (#373)
+Common: disable formatting on compile (#358)
+Common: add sbt publishLocal operation to test action (#357)
+Common: add toThrift and toRaw methods to CollectorPayload (#345)
+Common: replace deprecated constructors in EnrichedEventSpec (#354)
+Common: improve unit tests coverage (#335)
+Common: use test Maxmind databases (#350)
+Common: get rid of placeholder schema in enrichment configurations (#302)
+Common: move EnrichmentConf into its own module (#303)
+Common: get rid of Eval instances (#300)
+Common: add tests for Input for SQL enrichment (#316)
+
 Version 1.3.2 (2020-09-06)
 --------------------------
 Common: convert null fields in EnrichedEvent to None in PartiallyEnrichedEvent (#331)

build.sbt

@@ -19,7 +19,7 @@
 lazy val root = project.in(file("."))
   .settings(name := "enrich")
   .settings(BuildSettings.basicSettings)
-  .aggregate(common, beam, stream, kinesis, kafka, nsq, stdin, integrationTests)
+  .aggregate(common, beam, stream, kinesis, kafka, nsq, stdin, fs2)
 
 lazy val common = project
   .in(file("modules/common"))
@@ -54,8 +54,8 @@ lazy val common = project
       Dependencies.Libraries.scalaForex,
       Dependencies.Libraries.scalaWeather,
       Dependencies.Libraries.gatlingJsonpath,
-      Dependencies.Libraries.scalaLruMap,
       Dependencies.Libraries.badRows,
+      Dependencies.Libraries.igluClient,
       Dependencies.Libraries.snowplowRawEvent,
       Dependencies.Libraries.collectorPayload,
       Dependencies.Libraries.schemaSniffer,
@@ -130,7 +130,11 @@ lazy val nsq = project
   .settings(
     packageName in Docker := "snowplow/stream-enrich-nsq",
   )
-  .settings(libraryDependencies ++= Seq(Dependencies.Libraries.nsqClient))
+  .settings(libraryDependencies ++= Seq(
+    Dependencies.Libraries.log4j,
+    Dependencies.Libraries.log4jApi,
+    Dependencies.Libraries.nsqClient
+  ))
   .enablePlugins(JavaAppPackaging, DockerPlugin)
   .dependsOn(stream)
 
@@ -178,13 +182,59 @@ lazy val beam =
     )
     .enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
 
-lazy val integrationTests = project
-  .in(file("modules/integration-tests"))
-  .settings(moduleName := "integration-tests")
-  .settings(allStreamSettings)
+Global / onChangedBuildSource := ReloadOnSourceChanges
+
+lazy val fs2 = project
+  .in(file("modules/fs2"))
+  .dependsOn(common)
+  .settings(BuildSettings.basicSettings)
+  .settings(BuildSettings.formatting)
+  .settings(BuildSettings.scoverageSettings)
   .settings(BuildSettings.addExampleConfToTestCp)
-  .settings(libraryDependencies ++= Seq(
-    Dependencies.Libraries.kafka,
-    Dependencies.Libraries.jinJava
-  ))
-  .dependsOn(stream % "test->test", kafka % "test->compile")
+  .settings(BuildSettings.sbtAssemblySettings)
+  .settings(
+    name := "fs2-enrich",
+    description := "High-performance streaming Snowplow Enrich job built on top of functional streams",
+    buildInfoKeys := Seq[BuildInfoKey](organization, name, version, description),
+    buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.fs2.generated",
+    packageName in Docker := "snowplow/fs2-enrich",
+  )
+  .settings(parallelExecution in Test := false)
+  .settings(
+    libraryDependencies ++= Seq(
+      Dependencies.Libraries.decline,
+      Dependencies.Libraries.fs2PubSub,
+      Dependencies.Libraries.circeExtras,
+      Dependencies.Libraries.circeLiteral,
+      Dependencies.Libraries.circeConfig,
+      Dependencies.Libraries.catsEffect,
+      Dependencies.Libraries.fs2,
+      Dependencies.Libraries.fs2Io,
+      Dependencies.Libraries.slf4j,
+      Dependencies.Libraries.sentry,
+      Dependencies.Libraries.log4cats,
+      Dependencies.Libraries.catsRetry,
+      Dependencies.Libraries.http4sClient,
+      Dependencies.Libraries.fs2BlobS3,
+      Dependencies.Libraries.fs2BlobGcs,
+      Dependencies.Libraries.metrics,
+      Dependencies.Libraries.pureconfig.withRevision(Dependencies.V.pureconfig013),
+      Dependencies.Libraries.pureconfigCats.withRevision(Dependencies.V.pureconfig013),
+      Dependencies.Libraries.pureconfigCirce.withRevision(Dependencies.V.pureconfig013),
+      Dependencies.Libraries.specs2,
+      Dependencies.Libraries.specs2CE,
+      Dependencies.Libraries.scalacheck,
+      Dependencies.Libraries.specs2Scalacheck,
+      Dependencies.Libraries.http4sDsl,
+      Dependencies.Libraries.http4sServer
+    ),
+    addCompilerPlugin("com.olegpy" %% "better-monadic-for" % "0.3.1")
+  )
+  .enablePlugins(BuildInfoPlugin)
+  .settings(BuildSettings.dockerSettings)
+  .enablePlugins(BuildInfoPlugin, JavaAppPackaging, DockerPlugin)
+
+lazy val bench = project
+  .in(file("modules/bench"))
+  .dependsOn(fs2 % "test->test")
+  .enablePlugins(JmhPlugin)