Merge remote-tracking branch 'origin/main' into zhongzc/fix-fulltext-…

…exceed-buffet-limit
GreptimeTeam · Aug 16, 2024 · ece4c90 · ece4c90
2 parents 057e6d6 + ec59ce5
commit ece4c90
Show file tree

Hide file tree

Showing 137 changed files with 13,025 additions and 6,383 deletions.
diff --git a/.env.example b/.env.example
@@ -14,10 +14,11 @@ GT_AZBLOB_CONTAINER=AZBLOB container
 GT_AZBLOB_ACCOUNT_NAME=AZBLOB account name
 GT_AZBLOB_ACCOUNT_KEY=AZBLOB account key
 GT_AZBLOB_ENDPOINT=AZBLOB endpoint
-# Settings for gcs test 
-GT_GCS_BUCKET = GCS bucket 
+# Settings for gcs test
+GT_GCS_BUCKET = GCS bucket
 GT_GCS_SCOPE  = GCS scope
-GT_GCS_CREDENTIAL_PATH = GCS credential path 
+GT_GCS_CREDENTIAL_PATH = GCS credential path
+GT_GCS_CREDENTIAL = GCS credential
 GT_GCS_ENDPOINT = GCS end point
 # Settings for kafka wal test
 GT_KAFKA_ENDPOINTS = localhost:9092

diff --git a/.github/actions/setup-postgres-cluster/action.yml b/.github/actions/setup-postgres-cluster/action.yml
@@ -0,0 +1,30 @@
+name: Setup PostgreSQL
+description: Deploy PostgreSQL on Kubernetes
+inputs:
+  postgres-replicas:
+    default: 1
+    description: "Number of PostgreSQL replicas"
+  namespace:
+    default: "postgres-namespace"
+  postgres-version:
+    default: "14.2"
+    description: "PostgreSQL version"
+  storage-size:
+    default: "1Gi"
+    description: "Storage size for PostgreSQL"
+
+runs:
+  using: composite
+  steps:
+  - name: Install PostgreSQL
+    shell: bash
+    run: |
+      helm upgrade \
+        --install postgresql oci://registry-1.docker.io/bitnamicharts/postgresql \
+        --set replicaCount=${{ inputs.postgres-replicas }} \
+        --set image.tag=${{ inputs.postgres-version }} \
+        --set persistence.size=${{ inputs.storage-size }} \
+        --set postgresql.username=greptimedb \
+        --set postgresql.password=admin \
+        --create-namespace \
+        -n ${{ inputs.namespace }}
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -338,6 +338,8 @@ jobs:
         uses: ./.github/actions/setup-kafka-cluster
       - name: Setup Etcd cluser
         uses: ./.github/actions/setup-etcd-cluster
+      - name: Setup Postgres cluser
+        uses: ./.github/actions/setup-postgres-cluster
       # Prepares for fuzz tests
       - uses: arduino/setup-protoc@v3
         with:
@@ -476,6 +478,8 @@ jobs:
         uses: ./.github/actions/setup-kafka-cluster
       - name: Setup Etcd cluser
         uses: ./.github/actions/setup-etcd-cluster
+      - name: Setup Postgres cluser
+        uses: ./.github/actions/setup-postgres-cluster
       # Prepares for fuzz tests
       - uses: arduino/setup-protoc@v3
         with:
@@ -702,6 +706,9 @@ jobs:
       - name: Setup minio
         working-directory: tests-integration/fixtures/minio
         run: docker compose -f docker-compose-standalone.yml up -d --wait
+      - name: Setup postgres server
+        working-directory: tests-integration/fixtures/postgres
+        run: docker compose -f docker-compose-standalone.yml up -d --wait
       - name: Run nextest cases
         run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
         env:
@@ -718,6 +725,7 @@ jobs:
           GT_MINIO_REGION: us-west-2
           GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
+          GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
           GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
           GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
           UNITTEST_LOG_DIR: "__unittest_logs"

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -124,7 +124,7 @@ humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "049171eb16cb4249d8099751a0c46750d1fe88e7" }
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd" }
 mockall = "0.11.4"
 moka = "0.12"
 notify = "6.1"
@@ -174,6 +174,7 @@ sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "5
 strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.36", features = ["full"] }
+tokio-postgres = "0.7"
 tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
@@ -243,7 +244,7 @@ table = { path = "src/table" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
-rev = "049171eb16cb4249d8099751a0c46750d1fe88e7"
+rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
 
 [profile.release]
 debug = 1

diff --git a/config/config.md b/config/config.md
@@ -98,6 +98,7 @@
 | `storage.account_key` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.scope` | String | `None` | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.credential_path` | String | `None` | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential` | String | `None` | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.container` | String | `None` | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.sas_token` | String | `None` | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.endpoint` | String | `None` | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
@@ -387,6 +388,7 @@
 | `storage.account_key` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.scope` | String | `None` | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.credential_path` | String | `None` | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential` | String | `None` | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
 | `storage.container` | String | `None` | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.sas_token` | String | `None` | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
 | `storage.endpoint` | String | `None` | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
@@ -241,6 +241,7 @@ backoff_deadline = "5mins"
 # root = "data"
 # scope = "test"
 # credential_path = "123456"
+# credential = "base64-credential"
 # endpoint = "https://storage.googleapis.com"
 
 ## The data storage options.
@@ -312,6 +313,11 @@ scope = "test"
 ## +toml2docs:none-default
 credential_path = "test"
 
+## The credential of the google cloud storage.
+## **It's only used when the storage type is `Gcs`**.
+## +toml2docs:none-default
+credential= "base64-credential"
+
 ## The container of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
 ## +toml2docs:none-default

diff --git a/config/standalone.example.toml b/config/standalone.example.toml
@@ -286,6 +286,7 @@ retry_delay = "500ms"
 # root = "data"
 # scope = "test"
 # credential_path = "123456"
+# credential = "base64-credential"
 # endpoint = "https://storage.googleapis.com"
 
 ## The data storage options.
@@ -357,6 +358,11 @@ scope = "test"
 ## +toml2docs:none-default
 credential_path = "test"
 
+## The credential of the google cloud storage.
+## **It's only used when the storage type is `Gcs`**.
+## +toml2docs:none-default
+credential = "base64-credential"
+
 ## The container of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
 ## +toml2docs:none-default

diff --git a/grafana/README.md b/grafana/README.md
@@ -25,7 +25,7 @@ Please ensure the following configuration before importing the dashboard into Gr
 
 __1. Prometheus scrape config__
 
-Assign `greptime_pod` label to each host target. We use this label to identify each node instance.
+Configure Prometheus to scrape the cluster.
 
 ```yml
 # example config
@@ -34,27 +34,15 @@ Assign `greptime_pod` label to each host target. We use this label to identify e
 scrape_configs:
   - job_name: metasrv
     static_configs:
-    - targets: ['<ip>:<port>']
-      labels:
-        greptime_pod: metasrv
+    - targets: ['<metasrv-ip>:<port>']
 
   - job_name: datanode
     static_configs:
-    - targets: ['<ip>:<port>']
-      labels:
-        greptime_pod: datanode1
-    - targets: ['<ip>:<port>']
-      labels:
-        greptime_pod: datanode2
-    - targets: ['<ip>:<port>']
-      labels:
-        greptime_pod: datanode3
+    - targets: ['<datanode0-ip>:<port>', '<datanode1-ip>:<port>', '<datanode2-ip>:<port>']
 
   - job_name: frontend
     static_configs:
-    - targets: ['<ip>:<port>']
-      labels:
-        greptime_pod: frontend
+    - targets: ['<frontend-ip>:<port>']
 ```
 
 __2. Grafana config__
@@ -63,4 +51,4 @@ Create a Prometheus data source in Grafana before using this dashboard. We use `
 
 ### Usage
 
-Use `datasource` or `greptime_pod` on the upper-left corner to filter data from certain node.
+Use `datasource` or `instance` on the upper-left corner to filter data from certain node.