PubSub: drop attributes that are longer than 1024 characters (close #485

)
snowplow · Aug 9, 2021 · f533f65 · f533f65
1 parent f9b71f0
commit f533f65
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Environment.scala b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Environment.scala
@@ -261,8 +261,11 @@ object Environment {
  }
 
  private def attributesFromFields(fields: Map[String, Field])(ee: EnrichedEvent): Map[String, String] =
- fields.flatMap {
- case (k, f) => Option(f.get(ee)).map(v => k -> v.toString)
- }
+ fields
+ .flatMap {
+ case (k, f) => Option(f.get(ee)).map(v => k -> v.toString)
+ }
+ .filterNot(_._2.length > ATTRIBUTE_MAXLENGTH)
 
+ private final val ATTRIBUTE_MAXLENGTH = 1024
 }
diff --git a/.../src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/config/EnvironmentSpec.scala b/.../src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/config/EnvironmentSpec.scala
@@ -22,10 +22,11 @@ import org.specs2.mutable.Specification
 class EnvironmentSpec extends Specification with CatsIO {
 
  "outputAttributes" should {
- "fetch attribute values" in {
- val output = io.Output.PubSub("projects/test-project/topics/good-topic", Some(Set("app_id")), None, None, None, None)
+ "fetch attribute values shorter than 1024 characters" in {
+ val output = io.Output.PubSub("projects/test-project/topics/good-topic", Some(Set("app_id", "platform")), None, None, None, None)
  val ee = new EnrichedEvent()
  ee.app_id = "test_app"
+ ee.platform = "a".repeat(1025)
 
  val result = Environment.outputAttributes(output)(ee)