Kind #7582
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Kind | |
| # Any change in triggers needs to be reflected in the concurrency group. | |
| on: | |
| pull_request: {} | |
| schedule: | |
| - cron: '30 */2 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || 'scheduled' }} | |
| cancel-in-progress: true | |
| env: | |
| # renovate: datasource=github-releases depName=kubernetes-sigs/kind | |
| kind_version: v0.20.0 | |
| kind_config: .github/kind-config.yaml | |
| TIMEOUT: 2m | |
| LOG_TIME: 30m | |
| # renovate: datasource=github-releases depName=cilium/cilium | |
| cilium_version: v1.14.0 | |
| kubectl_version: v1.23.6 | |
| jobs: | |
| installation-and-connectivity: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 50 | |
| strategy: | |
| matrix: | |
| mode: ["classic", "helm"] | |
| steps: | |
| - name: Set mode | |
| run: | | |
| echo "CILIUM_CLI_MODE=${{ matrix.mode }}" >> $GITHUB_ENV | |
| - name: Checkout | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | |
| - name: Install kubectl | |
| run: | | |
| curl -sLO "https://dl.k8s.io/release/${{ env.kubectl_version }}/bin/linux/amd64/kubectl" | |
| curl -sLO "https://dl.k8s.io/${{ env.kubectl_version }}/bin/linux/amd64/kubectl.sha256" | |
| echo "$(cat kubectl.sha256) kubectl" | sha256sum --check | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| kubectl version --client | |
| - name: Set up Go | |
| uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 | |
| with: | |
| # renovate: datasource=golang-version depName=go | |
| go-version: 1.20.6 | |
| - name: Set up Go for root | |
| run: | | |
| sudo ln -sf `which go` `sudo which go` || true | |
| sudo go version | |
| - name: Build and install cilium CLI binary | |
| run: sudo make install | |
| - name: Create kind cluster | |
| uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 | |
| with: | |
| version: ${{ env.kind_version }} | |
| config: ${{ env.kind_config }} | |
| - name: Set NODES_WITHOUT_CILIUM | |
| run: | | |
| # To add more elements, keep it comma-separated. | |
| echo "NODES_WITHOUT_CILIUM=chart-testing-worker2,chart-testing-worker3" >> $GITHUB_ENV | |
| # Install Cilium with HostPort support and enables Prometheus for extended connectivity test. | |
| - name: Install Cilium | |
| run: | | |
| cilium install \ | |
| --version=${{ env.cilium_version }} \ | |
| --nodes-without-cilium="${NODES_WITHOUT_CILIUM}" \ | |
| --wait=false \ | |
| --helm-set bpf.monitorAggregation=none \ | |
| --helm-set cni.chainingMode=portmap \ | |
| --helm-set loadBalancer.l7.backend=envoy \ | |
| --helm-set tls.secretsBackend=k8s \ | |
| --helm-set prometheus.enabled=true | |
| - name: Enable Relay | |
| run: | | |
| cilium hubble enable --ui | |
| cilium status --wait | |
| - name: Relay Port Forward | |
| run: | | |
| cilium hubble port-forward& | |
| sleep 10s | |
| [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]] | |
| - name: Set up external targets | |
| id: external_targets | |
| run: | | |
| export worker2_ip=$(kubectl get nodes chart-testing-worker2 -o=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') | |
| export worker3_ip=$(kubectl get nodes chart-testing-worker3 -o=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') | |
| echo "worker2_ip=${worker2_ip}" >> $GITHUB_OUTPUT | |
| echo "worker3_ip=${worker3_ip}" >> $GITHUB_OUTPUT | |
| kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml | |
| kubectl rollout status -n cert-manager deployment.apps/cert-manager | |
| kubectl rollout status -n cert-manager deployment.apps/cert-manager-webhook | |
| kubectl create ns external-targets | |
| cat .github/external-targets/certs.yaml | envsubst | kubectl apply -n external-targets -f - | |
| until kubectl get secret -n external-targets external-target-cert &> /dev/null; do sleep 1s; done | |
| kubectl apply -n external-targets -f .github/external-targets/nginx.yaml | |
| kubectl rollout status -n external-targets ds/nginx | |
| - name: Connectivity Test | |
| run: | | |
| # Run the connectivity test in non-default namespace (i.e. not cilium-test) | |
| cilium connectivity test --debug --all-flows --test-namespace test-namespace \ | |
| --include-unsafe-tests \ | |
| --collect-sysdump-on-failure --junit-file cilium-junit-${{ matrix.mode }}-1.xml \ | |
| --junit-property mode=${{ matrix.mode }} --junit-property type=no-tunnel \ | |
| --curl-insecure \ | |
| --external-target chart-testing-worker2 \ | |
| --external-target-ca-namespace=external-targets \ | |
| --external-target-ca-name=ca \ | |
| --external-cidr 172.18.0.0/16 \ | |
| --external-ip ${{ steps.external_targets.outputs.worker2_ip }} \ | |
| --external-other-ip ${{ steps.external_targets.outputs.worker3_ip }} | |
| - name: Uninstall cilium | |
| run: | | |
| pkill -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" || test $? -eq 1 | |
| cilium uninstall --wait | |
| - name: Install Cilium with IPsec Encryption | |
| if: ${{ matrix.mode == 'classic' }} | |
| run: | | |
| cilium install \ | |
| --version=${{ env.cilium_version}} \ | |
| --encryption=ipsec \ | |
| --nodes-without-cilium="${NODES_WITHOUT_CILIUM}" \ | |
| --helm-set kubeProxyReplacement=disabled | |
| - name: Install Cilium with IPsec Encryption | |
| if: ${{ matrix.mode == 'helm' }} | |
| run: | | |
| kubectl create -n kube-system secret generic cilium-ipsec-keys \ | |
| --from-literal=keys="3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128" | |
| cilium install \ | |
| --version=${{ env.cilium_version}} \ | |
| --nodes-without-cilium="${NODES_WITHOUT_CILIUM}" \ | |
| --helm-set encryption.enabled=true \ | |
| --helm-set encryption.type=ipsec \ | |
| --helm-set kubeProxyReplacement=disabled | |
| - name: Enable Relay | |
| run: | | |
| cilium hubble enable | |
| cilium status --wait | |
| - name: Relay Port Forward | |
| run: | | |
| cilium hubble port-forward& | |
| sleep 10s | |
| [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]] | |
| - name: Connectivity test | |
| run: | | |
| cilium connectivity test --debug --force-deploy --all-flows --test-namespace test-namespace \ | |
| --include-unsafe-tests \ | |
| --collect-sysdump-on-failure --junit-file cilium-junit-${{ matrix.mode }}-2.xml \ | |
| --junit-property mode=${{ matrix.mode }} --junit-property type=ipsec \ | |
| --curl-insecure \ | |
| --external-target chart-testing-worker2 \ | |
| --external-target-ca-namespace=external-targets \ | |
| --external-target-ca-name=ca \ | |
| --external-cidr 172.18.0.0/16 \ | |
| --external-ip ${{ steps.external_targets.outputs.worker2_ip }} \ | |
| --external-other-ip ${{ steps.external_targets.outputs.worker3_ip }} | |
| - name: Upload JUnit | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 | |
| with: | |
| name: cilium-junits | |
| path: cilium-junit*.xml | |
| retention-days: 2 | |
| - name: Cleanup | |
| if: ${{ always() }} | |
| run: | | |
| cilium status | |
| kubectl get pods --all-namespaces -o wide | |
| cilium sysdump --output-filename cilium-sysdump-out --hubble-flows-count 10000 | |
| shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently | |
| - name: Unlabel nodes | |
| if: ${{ matrix.mode == 'helm' }} | |
| run: | | |
| IFS=',' read -ra nodes <<< "$NODES_WITHOUT_CILIUM" | |
| for node in "${nodes[@]}"; do | |
| kubectl label nodes "${node}" cilium.io/no-schedule- | |
| done | |
| - name: Upload sysdump | |
| if: ${{ !success() }} | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 | |
| with: | |
| name: cilium-sysdumps | |
| path: cilium-sysdump-*.zip | |
| retention-days: 5 | |
| helm-upgrade-clustermesh: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 50 | |
| env: | |
| CILIUM_CLI_MODE: helm | |
| kind_config_1: .github/kind-config-1.yaml | |
| kind_config_2: .github/kind-config-2.yaml | |
| # helm/kind-action will override the "name:" provided in the kind config with "chart-testing" unless these are | |
| # specified as inputs. These must also match the suffix here for CLUSTER1 and CLUSTER2. | |
| CLUSTER_NAME_1: c.1 | |
| CLUSTER_NAME_2: c.2 | |
| CLUSTER1: kind-c.1 | |
| CLUSTER2: kind-c.2 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | |
| - name: Install kubectl | |
| run: | | |
| curl -sLO "https://dl.k8s.io/release/${{ env.kubectl_version }}/bin/linux/amd64/kubectl" | |
| curl -sLO "https://dl.k8s.io/${{ env.kubectl_version }}/bin/linux/amd64/kubectl.sha256" | |
| echo "$(cat kubectl.sha256) kubectl" | sha256sum --check | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| kubectl version --client | |
| - name: Set up Go | |
| uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 | |
| with: | |
| # renovate: datasource=golang-version depName=go | |
| go-version: 1.20.6 | |
| - name: Set up Go for root | |
| run: | | |
| sudo ln -sf `which go` `sudo which go` || true | |
| sudo go version | |
| - name: Build and install cilium CLI binary | |
| run: sudo make install | |
| - name: Create kind cluster 1 | |
| uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 | |
| with: | |
| version: ${{ env.kind_version }} | |
| config: ${{ env.kind_config_1 }} | |
| cluster_name: ${{ env.CLUSTER_NAME_1 }} | |
| - name: Install Cilium on cluster 1 | |
| run: | | |
| cilium install --context $CLUSTER1 \ | |
| --version=${{ env.cilium_version }} \ | |
| --wait=true \ | |
| --helm-set bpf.monitorAggregation=none \ | |
| --helm-set cni.chainingMode=portmap \ | |
| --helm-set cluster.id=1 \ | |
| --helm-set cluster.name=$CLUSTER1 | |
| - name: Create kind cluster 2 | |
| uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 | |
| with: | |
| version: ${{ env.kind_version }} | |
| config: ${{ env.kind_config_2 }} | |
| cluster_name: ${{ env.CLUSTER_NAME_2 }} | |
| - name: Install Cilium on cluster 2 | |
| run: | | |
| cilium install --context $CLUSTER2 \ | |
| --version=${{ env.cilium_version }} \ | |
| --wait=true \ | |
| --helm-set bpf.monitorAggregation=none \ | |
| --helm-set cni.chainingMode=portmap \ | |
| --helm-set cluster.id=2 \ | |
| --helm-set cluster.name=$CLUSTER2 | |
| - name: Enable ClusterMesh on cluster 1 using helm-based upgrade | |
| run: | | |
| cilium upgrade --reuse-values --context $CLUSTER1 \ | |
| --wait=true \ | |
| --helm-set clustermesh.useAPIServer=true \ | |
| --helm-set clustermesh.apiserver.service.type=NodePort \ | |
| --helm-set clustermesh.apiserver.tls.server.extraDnsNames={"$CLUSTER1.mesh.cilium.io,$CLUSTER2.mesh.cilium.io"} | |
| - name: Copy CA cert from cluster 1 to cluster 2 | |
| run: | | |
| kubectl --context $CLUSTER2 delete secret -n kube-system cilium-ca && \ | |
| kubectl --context $CLUSTER1 get secrets -n kube-system cilium-ca -oyaml \ | |
| | kubectl --context $CLUSTER2 apply -f - | |
| # Restart Cilium on cluster 2 | |
| kubectl --context $CLUSTER2 delete pod -l app.kubernetes.io/part-of=cilium -A | |
| - name: Enable ClusterMesh on cluster 2 using helm-based upgrade | |
| run: | | |
| cilium upgrade --reuse-values --context $CLUSTER2 \ | |
| --wait=true \ | |
| --helm-set clustermesh.useAPIServer=true \ | |
| --helm-set clustermesh.apiserver.service.type=NodePort \ | |
| --helm-set clustermesh.apiserver.tls.server.extraDnsNames={"$CLUSTER1.mesh.cilium.io,$CLUSTER2.mesh.cilium.io"} | |
| - name: Rename the secrets expected by the clustermesh connect command | |
| run: | | |
| kubectl get secrets --context $CLUSTER1 \ | |
| -n kube-system clustermesh-apiserver-remote-cert -oyaml \ | |
| | sed 's/name: .*/name: clustermesh-apiserver-client-cert/' \ | |
| | kubectl apply --context $CLUSTER1 -f - | |
| kubectl get secrets --context $CLUSTER2 \ | |
| -n kube-system clustermesh-apiserver-remote-cert -oyaml \ | |
| | sed 's/name: .*/name: clustermesh-apiserver-client-cert/' \ | |
| | kubectl apply --context $CLUSTER2 -f - | |
| - name: Connect the two clusters using clustermesh | |
| run: | | |
| cilium clustermesh connect --context $CLUSTER1 --destination-context $CLUSTER2 | |
| cilium clustermesh status --context $CLUSTER1 --wait | |
| - name: Run the multicluster connectivity tests | |
| run: | | |
| cilium connectivity test --context $CLUSTER1 --multi-cluster $CLUSTER2 --debug \ | |
| --include-unsafe-tests \ | |
| --collect-sysdump-on-failure --junit-file cilium-junit-clustermesh-1.xml \ | |
| --junit-property mode=clustermesh --junit-property type=ipsec | |
| - name: Upload JUnit | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 | |
| with: | |
| name: cilium-junits | |
| path: cilium-junit*.xml | |
| retention-days: 2 | |
| - name: Cleanup | |
| if: ${{ always() }} | |
| run: | | |
| cilium --context $CLUSTER1 status | |
| kubectl --context $CLUSTER1 get pods --all-namespaces -o wide | |
| cilium --context $CLUSTER1 sysdump --output-filename cilium-sysdump-out-c1 | |
| cilium --context $CLUSTER2 status | |
| kubectl --context $CLUSTER2 get pods --all-namespaces -o wide | |
| cilium --context $CLUSTER2 sysdump --output-filename cilium-sysdump-out-c2 | |
| shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently | |
| - name: Upload sysdump from cluster 1 | |
| if: ${{ !success() }} | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 | |
| with: | |
| name: cilium-sysdump-out-c1.zip | |
| path: cilium-sysdump-out-c1.zip | |
| retention-days: 5 | |
| - name: Upload sysdump from cluster 2 | |
| if: ${{ !success() }} | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 | |
| with: | |
| name: cilium-sysdump-out-c2.zip | |
| path: cilium-sysdump-out-c2.zip | |
| retention-days: 5 |