Add experimental partitions ring lifecycler support (#7349)

* Add experimental partitions ring lifecycler support Signed-off-by: Marco Pracucci <marco@pracucci.com> Co-authored-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com> Co-authored-by: Peter Štibraný <pstibrany@gmail.com> * Use http.Error() Signed-off-by: Marco Pracucci <marco@pracucci.com> * Skip deprecated linter check for code that will be soon replaced Signed-off-by: Marco Pracucci <marco@pracucci.com> * Updated dskit Signed-off-by: Marco Pracucci <marco@pracucci.com> * Add partition id to logs Signed-off-by: Marco Pracucci <marco@pracucci.com> * Added comment Signed-off-by: Marco Pracucci <marco@pracucci.com> * Added TestIngester_ShouldNotCreatePartitionIfThereIsShutdownMarker Signed-off-by: Marco Pracucci <marco@pracucci.com> --------- Signed-off-by: Marco Pracucci <marco@pracucci.com> Co-authored-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com> Co-authored-by: Peter Štibraný <pstibrany@gmail.com>
grafana · Feb 13, 2024 · 2b34ad3 · 2b34ad3
1 parent 18b6faf
commit 2b34ad3
Show file tree

Hide file tree

Showing 31 changed files with 1,259 additions and 119 deletions.
diff --git a/...st-storage/.data-mimir-write-1/.gitignore → ...age/.data-mimir-write-zone-a-1/.gitignore b/...st-storage/.data-mimir-write-1/.gitignore → ...age/.data-mimir-write-zone-a-1/.gitignore
diff --git a/...st-storage/.data-mimir-write-2/.gitignore → ...age/.data-mimir-write-zone-a-2/.gitignore b/...st-storage/.data-mimir-write-2/.gitignore → ...age/.data-mimir-write-zone-a-2/.gitignore
diff --git a/...st-storage/.data-mimir-write-3/.gitignore → ...age/.data-mimir-write-zone-a-3/.gitignore b/...st-storage/.data-mimir-write-3/.gitignore → ...age/.data-mimir-write-zone-a-3/.gitignore
diff --git a/development/mimir-ingest-storage/.data-mimir-write-zone-b-1/.gitignore b/development/mimir-ingest-storage/.data-mimir-write-zone-b-1/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/development/mimir-ingest-storage/.data-mimir-write-zone-b-2/.gitignore b/development/mimir-ingest-storage/.data-mimir-write-zone-b-2/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/development/mimir-ingest-storage/.data-mimir-write-zone-b-3/.gitignore b/development/mimir-ingest-storage/.data-mimir-write-zone-b-3/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/development/mimir-ingest-storage/.dockerignore b/development/mimir-ingest-storage/.dockerignore
@@ -1,4 +1,7 @@
-.data-mimir-write-1
-.data-mimir-write-2
-.data-mimir-write-3
+.data-mimir-write-zone-a-1
+.data-mimir-write-zone-b-1
+.data-mimir-write-zone-a-2
+.data-mimir-write-zone-b-2
+.data-mimir-write-zone-a-3
+.data-mimir-write-zone-b-3
 .data-minio
diff --git a/development/mimir-ingest-storage/compose-up.sh b/development/mimir-ingest-storage/compose-up.sh
@@ -21,5 +21,5 @@ cd "$SCRIPT_DIR" && make
 # -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
 # GOARCH is not changed.
 CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/mimir "${SCRIPT_DIR}"/../../cmd/mimir
-docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" mimir-write-1
+docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" mimir-write-zone-a-1
 docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml up "$@"
diff --git a/development/mimir-ingest-storage/compose-update-mimir.sh b/development/mimir-ingest-storage/compose-update-mimir.sh
@@ -22,6 +22,6 @@ cd "$SCRIPT_DIR" && make
 # GOARCH is not changed.
 CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/mimir "${SCRIPT_DIR}"/../../cmd/mimir
 # Build docker image
-docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" mimir-write-1
+docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" mimir-write-zone-a-1
 # Recreate Mimir containers
 docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml up -d --force-recreate "$@" mimir-write-{1..3} mimir-read-{1..2} mimir-backend-{1..2}
diff --git a/development/mimir-ingest-storage/config/grafana-agent.yaml b/development/mimir-ingest-storage/config/grafana-agent.yaml
@@ -10,7 +10,13 @@ prometheus:
       scrape_configs:
         - job_name: mimir-read-write-mode/mimir-write
           static_configs:
-            - targets: ['mimir-write-1:8080', 'mimir-write-2:8080', 'mimir-write-3:8080']
+            - targets:
+              - 'mimir-write-zone-a-1:8080'
+              - 'mimir-write-zone-b-1:8080'
+              - 'mimir-write-zone-a-2:8080'
+              - 'mimir-write-zone-b-2:8080'
+              - 'mimir-write-zone-a-3:8080'
+              - 'mimir-write-zone-b-3:8080'
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-read-write-mode'
@@ -31,4 +37,4 @@ prometheus:
                 container: 'mimir-backend'
 
       remote_write:
-        - url: http://mimir-write-1:8080/api/v1/push
+        - url: http://mimir-write-zone-a-1:8080/api/v1/push
diff --git a/development/mimir-ingest-storage/config/mimir.yaml b/development/mimir-ingest-storage/config/mimir.yaml
@@ -18,6 +18,11 @@ ingest_storage:
 ingester:
   return_only_grpc_errors: true
 
+  partition_ring:
+    min_partition_owners_count:       2
+    min_partition_owners_duration:    10s
+    delete_inactive_partition_after:  1m
+
 blocks_storage:
   s3:
     bucket_name:       mimir-blocks
@@ -43,7 +48,13 @@ blocks_storage:
 
 memberlist:
   # Use write replicas as seed nodes.
-  join_members: [ mimir-write-1, mimir-write-2, mimir-write-3 ]
+  join_members:
+    - mimir-write-zone-a-1
+    - mimir-write-zone-b-1
+    - mimir-write-zone-a-2
+    - mimir-write-zone-b-2
+    - mimir-write-zone-a-3
+    - mimir-write-zone-b-3
 
 ruler:
   rule_path: /data/ruler
@@ -82,6 +93,7 @@ overrides_exporter:
 
 limits:
   native_histograms_ingestion_enabled: true
+  ingestion_rate: 100000
 
 runtime_config:
   file: ./config/runtime.yaml
diff --git a/development/mimir-ingest-storage/docker-compose.jsonnet b/development/mimir-ingest-storage/docker-compose.jsonnet
@@ -14,23 +14,50 @@ std.manifestYamlDoc({
     {},
 
   write:: {
-    'mimir-write-1': mimirService({
-      name: 'mimir-write-1',
+    // Zone-a.
+    'mimir-write-zone-a-1': mimirService({
+      name: 'mimir-write-zone-a-1',
       target: 'write',
       publishedHttpPort: 8001,
-      extraVolumes: ['.data-mimir-write-1:/data:delegated'],
+      extraArguments: ['-ingester.ring.instance-availability-zone=zone-a'],
+      extraVolumes: ['.data-mimir-write-zone-a-1:/data:delegated'],
     }),
-    'mimir-write-2': mimirService({
-      name: 'mimir-write-2',
+    'mimir-write-zone-a-2': mimirService({
+      name: 'mimir-write-zone-a-2',
       target: 'write',
       publishedHttpPort: 8002,
-      extraVolumes: ['.data-mimir-write-2:/data:delegated'],
+      extraArguments: ['-ingester.ring.instance-availability-zone=zone-a'],
+      extraVolumes: ['.data-mimir-write-zone-a-2:/data:delegated'],
     }),
-    'mimir-write-3': mimirService({
-      name: 'mimir-write-3',
+    'mimir-write-zone-a-3': mimirService({
+      name: 'mimir-write-zone-a-3',
       target: 'write',
       publishedHttpPort: 8003,
-      extraVolumes: ['.data-mimir-write-3:/data:delegated'],
+      extraArguments: ['-ingester.ring.instance-availability-zone=zone-a'],
+      extraVolumes: ['.data-mimir-write-zone-a-3:/data:delegated'],
+    }),
+
+    // Zone-b.
+    'mimir-write-zone-b-1': mimirService({
+      name: 'mimir-write-zone-b-1',
+      target: 'write',
+      publishedHttpPort: 8011,
+      extraArguments: ['-ingester.ring.instance-availability-zone=zone-b'],
+      extraVolumes: ['.data-mimir-write-zone-b-1:/data:delegated'],
+    }),
+    'mimir-write-zone-b-2': mimirService({
+      name: 'mimir-write-zone-b-2',
+      target: 'write',
+      publishedHttpPort: 8012,
+      extraArguments: ['-ingester.ring.instance-availability-zone=zone-b'],
+      extraVolumes: ['.data-mimir-write-zone-b-2:/data:delegated'],
+    }),
+    'mimir-write-zone-b-3': mimirService({
+      name: 'mimir-write-zone-b-3',
+      target: 'write',
+      publishedHttpPort: 8013,
+      extraArguments: ['-ingester.ring.instance-availability-zone=zone-b'],
+      extraVolumes: ['.data-mimir-write-zone-b-3:/data:delegated'],
     }),
   },
 
@@ -176,6 +203,7 @@ std.manifestYamlDoc({
         kafka: { condition: 'service_healthy' },
       },
       env: {},
+      extraArguments: [],
       extraVolumes: [],
       memberlistBindPort: self.publishedHttpPort + 2000,
     },
@@ -192,7 +220,7 @@ std.manifestYamlDoc({
       '-config.file=./config/mimir.yaml' % options,
       '-target=%(target)s' % options,
       '-activity-tracker.filepath=/activity/%(name)s' % options,
-    ],
+    ] + options.extraArguments,
     environment: [
       '%s=%s' % [key, options.env[key]]
       for key in std.objectFields(options.env)

diff --git a/development/mimir-ingest-storage/docker-compose.yml b/development/mimir-ingest-storage/docker-compose.yml
@@ -130,75 +130,150 @@
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
-  "mimir-write-1":
+  "mimir-write-zone-a-1":
     "build":
       "context": "."
       "dockerfile": "dev.dockerfile"
     "command":
       - "./mimir"
       - "-config.file=./config/mimir.yaml"
       - "-target=write"
-      - "-activity-tracker.filepath=/activity/mimir-write-1"
+      - "-activity-tracker.filepath=/activity/mimir-write-zone-a-1"
+      - "-ingester.ring.instance-availability-zone=zone-a"
     "depends_on":
       "kafka":
         "condition": "service_healthy"
       "minio":
         "condition": "service_started"
     "environment": []
-    "hostname": "mimir-write-1"
+    "hostname": "mimir-write-zone-a-1"
     "image": "mimir"
     "ports":
       - "8001:8080"
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
-      - ".data-mimir-write-1:/data:delegated"
-  "mimir-write-2":
+      - ".data-mimir-write-zone-a-1:/data:delegated"
+  "mimir-write-zone-a-2":
     "build":
       "context": "."
       "dockerfile": "dev.dockerfile"
     "command":
       - "./mimir"
       - "-config.file=./config/mimir.yaml"
       - "-target=write"
-      - "-activity-tracker.filepath=/activity/mimir-write-2"
+      - "-activity-tracker.filepath=/activity/mimir-write-zone-a-2"
+      - "-ingester.ring.instance-availability-zone=zone-a"
     "depends_on":
       "kafka":
         "condition": "service_healthy"
       "minio":
         "condition": "service_started"
     "environment": []
-    "hostname": "mimir-write-2"
+    "hostname": "mimir-write-zone-a-2"
     "image": "mimir"
     "ports":
       - "8002:8080"
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
-      - ".data-mimir-write-2:/data:delegated"
-  "mimir-write-3":
+      - ".data-mimir-write-zone-a-2:/data:delegated"
+  "mimir-write-zone-a-3":
     "build":
       "context": "."
       "dockerfile": "dev.dockerfile"
     "command":
       - "./mimir"
       - "-config.file=./config/mimir.yaml"
       - "-target=write"
-      - "-activity-tracker.filepath=/activity/mimir-write-3"
+      - "-activity-tracker.filepath=/activity/mimir-write-zone-a-3"
+      - "-ingester.ring.instance-availability-zone=zone-a"
     "depends_on":
       "kafka":
         "condition": "service_healthy"
       "minio":
         "condition": "service_started"
     "environment": []
-    "hostname": "mimir-write-3"
+    "hostname": "mimir-write-zone-a-3"
     "image": "mimir"
     "ports":
       - "8003:8080"
     "volumes":
       - "./config:/mimir/config"
       - "./activity:/activity"
-      - ".data-mimir-write-3:/data:delegated"
+      - ".data-mimir-write-zone-a-3:/data:delegated"
+  "mimir-write-zone-b-1":
+    "build":
+      "context": "."
+      "dockerfile": "dev.dockerfile"
+    "command":
+      - "./mimir"
+      - "-config.file=./config/mimir.yaml"
+      - "-target=write"
+      - "-activity-tracker.filepath=/activity/mimir-write-zone-b-1"
+      - "-ingester.ring.instance-availability-zone=zone-b"
+    "depends_on":
+      "kafka":
+        "condition": "service_healthy"
+      "minio":
+        "condition": "service_started"
+    "environment": []
+    "hostname": "mimir-write-zone-b-1"
+    "image": "mimir"
+    "ports":
+      - "8011:8080"
+    "volumes":
+      - "./config:/mimir/config"
+      - "./activity:/activity"
+      - ".data-mimir-write-zone-b-1:/data:delegated"
+  "mimir-write-zone-b-2":
+    "build":
+      "context": "."
+      "dockerfile": "dev.dockerfile"
+    "command":
+      - "./mimir"
+      - "-config.file=./config/mimir.yaml"
+      - "-target=write"
+      - "-activity-tracker.filepath=/activity/mimir-write-zone-b-2"
+      - "-ingester.ring.instance-availability-zone=zone-b"
+    "depends_on":
+      "kafka":
+        "condition": "service_healthy"
+      "minio":
+        "condition": "service_started"
+    "environment": []
+    "hostname": "mimir-write-zone-b-2"
+    "image": "mimir"
+    "ports":
+      - "8012:8080"
+    "volumes":
+      - "./config:/mimir/config"
+      - "./activity:/activity"
+      - ".data-mimir-write-zone-b-2:/data:delegated"
+  "mimir-write-zone-b-3":
+    "build":
+      "context": "."
+      "dockerfile": "dev.dockerfile"
+    "command":
+      - "./mimir"
+      - "-config.file=./config/mimir.yaml"
+      - "-target=write"
+      - "-activity-tracker.filepath=/activity/mimir-write-zone-b-3"
+      - "-ingester.ring.instance-availability-zone=zone-b"
+    "depends_on":
+      "kafka":
+        "condition": "service_healthy"
+      "minio":
+        "condition": "service_started"
+    "environment": []
+    "hostname": "mimir-write-zone-b-3"
+    "image": "mimir"
+    "ports":
+      - "8013:8080"
+    "volumes":
+      - "./config:/mimir/config"
+      - "./activity:/activity"
+      - ".data-mimir-write-zone-b-3:/data:delegated"
   "minio":
     "command":
       - "server"

diff --git a/go.mod b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/google/gopacket v1.1.19
 	github.com/gorilla/mux v1.8.1
-	github.com/grafana/dskit v0.0.0-20240208074945-f245b483eb15
+	github.com/grafana/dskit v0.0.0-20240213103939-80881aa4a62f
 	github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/json-iterator/go v1.1.12

diff --git a/go.sum b/go.sum
@@ -495,8 +495,8 @@ github.com/gosimple/slug v1.1.1 h1:fRu/digW+NMwBIP+RmviTK97Ho/bEj/C9swrCspN3D4=
 github.com/gosimple/slug v1.1.1/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0=
 github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc h1:PXZQA2WCxe85Tnn+WEvr8fDpfwibmEPgfgFEaC87G24=
 github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4=
-github.com/grafana/dskit v0.0.0-20240208074945-f245b483eb15 h1:+oY4HiyUyGFY/DrLDBc7PcRwC1dIztT899JUPNxGVGE=
-github.com/grafana/dskit v0.0.0-20240208074945-f245b483eb15/go.mod h1:x5DMwyr1kyirtHOxoFSZ7RnyOgHdGh03ZruupdPetQM=
+github.com/grafana/dskit v0.0.0-20240213103939-80881aa4a62f h1:SU2XpJOzuclXYls6LyMgmQhv2TVLe7Oj6UbezYxZeM0=
+github.com/grafana/dskit v0.0.0-20240213103939-80881aa4a62f/go.mod h1:x5DMwyr1kyirtHOxoFSZ7RnyOgHdGh03ZruupdPetQM=
 github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc h1:BW+LjKJDz0So5LI8UZfW5neWeKpSkWqhmGjQFzcFfLM=
 github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc/go.mod h1:JVmqPBe8A/pZWwRoJW5ZjyALeY5OXMzPl7LrVXOdZAI=
 github.com/grafana/goautoneg v0.0.0-20231010094147-47ce5e72a9ae h1:Yxbw9jKGJVC6qAK5Ubzzb/qZwM6rRMMqaDc/d4Vp3pM=

diff --git a/integration/configs.go b/integration/configs.go
@@ -247,6 +247,10 @@ blocks_storage:
 			// and faster integration tests.
 			"-ingest-storage.kafka.last-produced-offset-poll-interval": "50ms",
 			"-ingest-storage.kafka.last-produced-offset-retry-timeout": "1s",
+
+			// Do not wait before switching an INACTIVE partition to ACTIVE.
+			"-ingester.partition-ring.min-partition-owners-count":    "0",
+			"-ingester.partition-ring.min-partition-owners-duration": "0s",
 		}
 	}
 )