add development example setup

CHANGELOG.md

@@ -12,11 +12,13 @@
 * adds `healthcheck_timeout` option to all components to tweak the timeout of healthchecks
 * adds `desired_cluster_status` option to opensearch output to signal healthy cluster status
 * initially run health checks on setup for every configured component
+* make `imagePullPolicy` configurable for helm chart deployments
 
 
 ### Improvements
 
 * remove AutoRuleCorpusTester
+* add manual how to use local images with minikube example setup to documentation
 
 ### Bugfix
 

charts/logprep/templates/deployment.yaml

@@ -34,7 +34,7 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
-          imagePullPolicy: Always
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
           ports:
             {{- if .Values.exporter.enabled }}
             - name: exporter

charts/logprep/values.yaml

@@ -7,6 +7,7 @@ image:
   registry: ghcr.io
   repository: fkie-cad/logprep
   tag: py3.11-stable
+  pullPolicy: Always
 
 # The pod resources
 resources:

doc/source/examples/minikube.rst

@@ -15,25 +15,19 @@ with the following commands:
 .. code-block:: bash
     :caption: Install package prerequisites
 
-    sudo apt-get install -y \
-        apt-transport-https \
-        ca-certificates \
-        curl \
-        software-properties-common
+    sudo apt-get install -y apt-transport-https ca-certificates  curl software-properties-common
 
 .. code-block:: bash
     :caption: Install minikube
 
-    sudo curl -Lo /usr/local/bin/minikube \
-      https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+    sudo curl -Lo /usr/local/bin/minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
     
     sudo chmod +x /usr/local/bin/minikube
 
 .. code-block:: bash
     :caption: Install kubectl
 
-    sudo curl -Lo /usr/local/bin/kubectl \
-      "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    sudo curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
 
     sudo chmod +x /usr/local/bin/kubectl
 
@@ -56,8 +50,8 @@ with the following commands:
     minikube config set driver docker
     minikube config set cpus 16 
     minikube config set memory 16GB
-    minikube addons enable ingress
     minikube start
+    minikube addons enable ingress
 
 Deploy the example
 ------------------
@@ -125,4 +119,36 @@ Test the opensiem connector:
     }
     2024-07-17 11:15:35 301643 Generator  INFO    : Execution time: 0.067013 seconds
 
-open your browser and go to `http://dashboards.opensiem`_ to see the generated data in the opensearch dashboards.
+open your browser and go to `opensearch dashboard <http://dashboards.opensiem>`_ to see the generated data in the opensearch dashboards.
+
+
+Use local container images
+--------------------------
+
+If you want to use local logprep container images, you can build the images with the following commands:
+
+.. code-block:: bash
+    :caption: switch docker context to minikube in bash
+
+    eval $(minikube docker-env)
+
+for powershell:
+
+.. code-block:: powershell
+    :caption: switch docker context to minikube in powershell
+
+    (minikube docker-env).replace("export ", '$env:') | out-string | Invoke-Expression
+
+Then build the logprep image with the following command:
+
+.. code-block:: bash
+    :caption: build this image using the Dockerfile in the root of the repository
+
+    docker buildx build -t local/logprep:latest --build-arg PYTHON_VERSION=3.11 --build-arg LOGPREP_VERSION=dev .
+
+Then install the opensiem example using the local logprep image:
+
+.. code-block:: bash
+    :caption: use the local values file to deploy the opensiem example
+
+    helm install opensiem examples/k8s --values examples/k8s/values-dev.yaml

examples/k8s/values-dev.yaml

@@ -0,0 +1,126 @@
+connector:
+  image:
+    # point your docker context to minikube and build the image
+    # `eval $(minikube docker-env)` or `(minikube docker-env).replace("export ", '$env:') | out-string | Invoke-Expression`
+    # build this image using the Dockerfile in the root of the repository
+    # `docker buildx build -t local/logprep:latest --build-arg PYTHON_VERSION=3.11 --build-arg LOGPREP_VERSION=dev .`
+    registry: local
+    repository: logprep
+    tag: latest
+    pullPolicy: IfNotPresent
+  replicas: 1
+  secrets: {}
+  logger:
+    level: INFO
+  input:
+    type: http_input
+    message_backlog_size: 150000
+    collect_meta: True
+    metafield_name: "@metadata"
+    uvicorn_config:
+      host: 0.0.0.0
+      port: 9000
+      workers: 1
+      access_log: true
+      server_header: false
+      date_header: false
+      ws: none
+      interface: asgi3
+      backlog: 16384
+      timeout_keep_alive: 65
+    endpoints:
+      /auth-json: json
+      /json: json
+      /lab/123/(ABC|DEF)/pl.*: plaintext
+      /lab/123/ABC/auditlog: jsonl
+      /health: plaintext
+  output:
+    type: confluentkafka_output
+    topic: consumer
+    error_topic: errors
+    flush_timeout: 300
+    send_timeout: 0
+    kafka_config:
+      bootstrap.servers: opensiem-kafka:9092
+      compression.type: none
+      statistics.interval.ms: "60000"
+      queue.buffering.max.messages: "100000000"
+      queue.buffering.max.kbytes: "1048576"
+      queue.buffering.max.ms: "10000"
+      batch.size: "1000000"
+      request.required.acks: "-1"
+  ingress:
+    enabled: true
+
+## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/kafka/values.yaml`
+kafka:
+  listeners:
+    client:
+      protocol: PLAINTEXT
+  controller:
+    replicaCount: 3
+  metrics:
+    jmx:
+      enabled: true
+  provisioning:
+    enabled: true
+    replicationFactor: 1
+    numPartitions: 10
+    topics:
+      - name: consumer
+
+logprep:
+  image:
+    registry: local
+    repository: logprep
+    tag: latest
+    pullPolicy: IfNotPresent
+  logger:
+    level: INFO
+  input:
+    type: confluentkafka_input
+    topic: consumer
+    kafka_config:
+      bootstrap.servers: opensiem-kafka:9092
+      group.id: cgroup3
+      enable.auto.commit: "true"
+      auto.commit.interval.ms: "10000"
+      enable.auto.offset.store: "false"
+      queued.min.messages: "100000"
+      queued.max.messages.kbytes: "65536"
+      statistics.interval.ms: "60000"
+    preprocessing:
+      version_info_target_field: Logprep_version_info
+      log_arrival_time_target_field: event.ingested
+      hmac:
+        target: <RAW_MSG>
+        key: "thisisasecureandrandomkey"
+        output_field: Full_event
+  output:
+    type: opensearch_output
+    hosts:
+      - opensiem-opensearch:9200
+    default_index: processed
+    error_index: errors
+    timeout: 10000
+    message_backlog_size: 2500
+    parallel_bulk: true
+    flush_timeout: 60
+    max_retries: 3
+    chunk_size: 500
+    thread_count: 5
+    user: admin
+    secret: admin
+    desired_cluster_status: ["green", "yellow"]
+## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml`
+opensearch:
+  dashboards:
+    enabled: true
+  ingest:
+    replicaCount: 1
+  master:
+    replicaCount: 1
+  data:
+    replicaCount: 1
+  coordinating:
+    replicaCount: 1