Skip to content

Commit

Permalink
Use Different Images for Agent and Controller
Browse files Browse the repository at this point in the history
Modified the code to build separate images for antrea-agent and
antrea-controller, because there are many resources that are not
required by controller and are required by agent only, and unified
image for both creates a burden when starting antrea-controller and
thus it takes time to start. For this reason I have create separate
images for antrea-agent and antrea-controller.

Fixes antrea-io#5691.

Signed-off-by: Pulkit Jain <jainpu@vmware.com>
  • Loading branch information
Pulkit Jain committed Jan 9, 2024
1 parent 2bf7c80 commit fad0b8f
Show file tree
Hide file tree
Showing 26 changed files with 347 additions and 62 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,11 @@ jobs:
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
./hack/build-antrea-linux-all.sh --pull --push-base-images
docker tag antrea/antrea-ubuntu:latest antrea/antrea-ubuntu-amd64:latest
docker tag antrea/antrea-controller-ubuntu:latest antrea/antrea-controller-ubuntu-amd64:latest
docker tag antrea/antrea-agent-ubuntu:latest antrea/antrea-agent-ubuntu-amd64:latest
docker push antrea/antrea-ubuntu-amd64:latest
docker push antrea/antrea-controller-ubuntu-amd64:latest
docker push antrea/antrea-agent-ubuntu-amd64:latest
- name: Trigger Antrea arm builds and multi-arch manifest update
if: ${{ github.repository == 'antrea-io/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
uses: benc-uk/workflow-dispatch@v1
Expand Down
20 changes: 13 additions & 7 deletions .github/workflows/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
run: |
./hack/build-antrea-linux-all.sh --pull --coverage
- name: Save Antrea image to tarball
run: docker save -o antrea-ubuntu.tar antrea/antrea-ubuntu-coverage:latest
run: docker save -o antrea-ubuntu.tar antrea/antrea-agent-ubuntu-coverage:latest antrea/antrea-controller-ubuntu-coverage:latest
- name: Upload Antrea image for subsequent jobs
uses: actions/upload-artifact@v4
with:
Expand Down Expand Up @@ -488,7 +488,8 @@ jobs:
- name: Load Antrea image
run: |
docker load -i antrea-ubuntu.tar
docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest
docker tag antrea/antrea-agent-ubuntu-coverage:latest antrea/antrea-agent-ubuntu:latest
docker tag antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-controller-ubuntu:latest
- name: Install Kind
run: |
KIND_VERSION=$(head -n1 ./ci/kind/version)
Expand Down Expand Up @@ -533,7 +534,8 @@ jobs:
- name: Load Antrea image
run: |
docker load -i antrea-ubuntu.tar
docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest
docker tag antrea/antrea-agent-ubuntu-coverage:latest antrea/antrea-agent-ubuntu:latest
docker tag antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-controller-ubuntu:latest
- name: Install Kind
run: |
KIND_VERSION=$(head -n1 ./ci/kind/version)
Expand Down Expand Up @@ -578,7 +580,8 @@ jobs:
- name: Load Antrea image
run: |
docker load -i antrea-ubuntu.tar
docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest
docker tag antrea/antrea-agent-ubuntu-coverage:latest antrea/antrea-agent-ubuntu:latest
docker tag antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-controller-ubuntu:latest
- name: Install Kind
run: |
KIND_VERSION=$(head -n1 ./ci/kind/version)
Expand Down Expand Up @@ -623,7 +626,8 @@ jobs:
- name: Load Antrea image
run: |
docker load -i antrea-ubuntu.tar
docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest
docker tag antrea/antrea-agent-ubuntu-coverage:latest antrea/antrea-agent-ubuntu:latest
docker tag antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-controller-ubuntu:latest
- name: Install Kind
run: |
KIND_VERSION=$(head -n1 ./ci/kind/version)
Expand Down Expand Up @@ -668,7 +672,8 @@ jobs:
- name: Load Antrea image
run: |
docker load -i antrea-ubuntu.tar
docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest
docker tag antrea/antrea-agent-ubuntu-coverage:latest antrea/antrea-agent-ubuntu:latest
docker tag antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-controller-ubuntu:latest
- name: Install Kind
run: |
KIND_VERSION=$(head -n1 ./ci/kind/version)
Expand Down Expand Up @@ -710,7 +715,8 @@ jobs:
- name: Load Antrea image
run: |
docker load -i antrea-ubuntu.tar
docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest
docker tag antrea/antrea-agent-ubuntu-coverage:latest antrea/antrea-agent-ubuntu:latest
docker tag antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-controller-ubuntu:latest
- name: Install Kind
run: |
KIND_VERSION=$(head -n1 ./ci/kind/version)
Expand Down
40 changes: 40 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,26 @@ else
endif
docker tag antrea/antrea-ubuntu:$(DOCKER_IMG_VERSION) antrea/antrea-ubuntu

.PHONY: build-controller-ubuntu
build-controller-ubuntu:
@echo "===> Building antrea/antrea-controller-ubuntu Docker image <==="
ifneq ($(NO_PULL),)
docker build -t antrea/antrea-controller-ubuntu:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.controller.ubuntu $(DOCKER_BUILD_ARGS) .
else
docker build --pull -t antrea/antrea-controller-ubuntu:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.controller.ubuntu $(DOCKER_BUILD_ARGS) .
endif
docker tag antrea/antrea-controller-ubuntu:$(DOCKER_IMG_VERSION) antrea/antrea-controller-ubuntu

.PHONY: build-agent-ubuntu
build-agent-ubuntu:
@echo "===> Building antrea/antrea-agent-ubuntu Docker image <==="
ifneq ($(NO_PULL),)
docker build -t antrea/antrea-agent-ubuntu:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.agent.ubuntu $(DOCKER_BUILD_ARGS) .
else
docker build --pull -t antrea/antrea-agent-ubuntu:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.agent.ubuntu $(DOCKER_BUILD_ARGS) .
endif
docker tag antrea/antrea-agent-ubuntu:$(DOCKER_IMG_VERSION) antrea/antrea-agent-ubuntu

# Build bins in a golang container, and build the antrea-ubuntu Docker image.
.PHONY: build-ubuntu
build-ubuntu:
Expand Down Expand Up @@ -374,6 +394,26 @@ else
endif
docker tag antrea/antrea-ubuntu-coverage:$(DOCKER_IMG_VERSION) antrea/antrea-ubuntu-coverage

.PHONY: build-controller-ubuntu-coverage
build-controller-ubuntu-coverage:
@echo "===> Building Antrea bins and antrea/antrea-controller-ubuntu-coverage Docker image <==="
ifneq ($(NO_PULL),)
docker build -t antrea/antrea-controller-ubuntu-coverage:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.controller.build.coverage $(DOCKER_BUILD_ARGS) .
else
docker build --pull -t antrea/antrea-controller-ubuntu-coverage:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.controller.build.coverage $(DOCKER_BUILD_ARGS) .
endif
docker tag antrea/antrea-controller-ubuntu-coverage:$(DOCKER_IMG_VERSION) antrea/antrea-controller-ubuntu-coverage

.PHONY: build-agent-ubuntu-coverage
build-agent-ubuntu-coverage:
@echo "===> Building Antrea bins and antrea/antrea-agent-ubuntu-coverage Docker image <==="
ifneq ($(NO_PULL),)
docker build -t antrea/antrea-agent-ubuntu-coverage:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.agent.build.coverage $(DOCKER_BUILD_ARGS) .
else
docker build --pull -t antrea/antrea-agent-ubuntu-coverage:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.agent.build.coverage $(DOCKER_BUILD_ARGS) .
endif
docker tag antrea/antrea-agent-ubuntu-coverage:$(DOCKER_IMG_VERSION) antrea/antrea-agent-ubuntu-coverage

.PHONY: build-scale-simulator
build-scale-simulator:
@echo "===> Building simulator bin and antrea-ubuntu-simulator image"
Expand Down
4 changes: 3 additions & 1 deletion build/charts/antrea/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ Kubernetes: `>= 1.16.0-0`
| agent.priorityClassName | string | `"system-node-critical"` | Prority class to use for the antrea-agent Pods. |
| agent.tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoSchedule","operator":"Exists"},{"effect":"NoExecute","operator":"Exists"}]` | Tolerations for the antrea-agent Pods. |
| agent.updateStrategy | object | `{"type":"RollingUpdate"}` | Update strategy for the antrea-agent DaemonSet. |
| agentImage | object | `{"pullPolicy":"IfNotPresent","repository":"antrea/antrea-agent-ubuntu","tag":""}` | Container image to use for the antrea-agent component. |
| antreaProxy.defaultLoadBalancerMode | string | `"nat"` | Determines how external traffic is processed when it's load balanced across Nodes by default. It must be one of "nat" or "dsr". |
| antreaProxy.enable | bool | `true` | To disable AntreaProxy, set this to false. |
| antreaProxy.nodePortAddresses | list | `[]` | String array of values which specifies the host IPv4/IPv6 addresses for NodePort. By default, all host addresses are used. |
Expand Down Expand Up @@ -82,6 +83,7 @@ Kubernetes: `>= 1.16.0-0`
| controller.priorityClassName | string | `"system-cluster-critical"` | Prority class to use for the antrea-controller Pod. |
| controller.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "antrea-controller-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. |
| controller.tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"},{"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":0}]` | Tolerations for the antrea-controller Pod. |
| controllerImage | object | `{"pullPolicy":"IfNotPresent","repository":"antrea/antrea-controller-ubuntu","tag":""}` | Container image to use for the antrea-controller component. |
| defaultMTU | int | `0` | Default MTU to use for the host gateway interface and the network interface of each Pod. By default, antrea-agent will discover the MTU of the Node's primary interface and adjust it to accommodate for tunnel encapsulation overhead if applicable. |
| disableTXChecksumOffload | bool | `false` | Disable TX checksum offloading for container network interfaces. It's supposed to be set to true when the datapath doesn't support TX checksum offloading, which causes packets to be dropped due to bad checksum. It affects Pods running on Linux Nodes only. |
| dnsServerOverride | string | `""` | Address of DNS server, to override the kube-dns Service. It's used to resolve hostnames in a FQDN policy. |
Expand All @@ -95,7 +97,7 @@ Kubernetes: `>= 1.16.0-0`
| flowExporter.flowPollInterval | string | `"5s"` | Determines how often the flow exporter polls for new connections. |
| flowExporter.idleFlowExportTimeout | string | `"15s"` | timeout after which a flow record is sent to the collector for idle flows. |
| hostGateway | string | `"antrea-gw0"` | Name of the interface antrea-agent will create and use for host <-> Pod communication. |
| image | object | `{"pullPolicy":"IfNotPresent","repository":"antrea/antrea-ubuntu","tag":""}` | Container image to use for Antrea components. |
| image | object | `{}` | Container image to use for Antrea components. DEPRECATED: use agentImage and controllerImage instead. |
| ipsec.authenticationMode | string | `"psk"` | The authentication mode to use for IPsec. Must be one of "psk" or "cert". |
| ipsec.csrSigner.autoApprove | bool | `true` | Enable auto approval of Antrea signer for IPsec certificates. |
| ipsec.csrSigner.selfSignedCA | bool | `true` | Whether or not to use auto-generated self-signed CA. |
Expand Down
34 changes: 33 additions & 1 deletion build/charts/antrea/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,40 @@
{{- end }}
{{- end -}}

{{- define "antreaImage" -}}
{{- define "antreaAgentImageTag" -}}
{{- if .Values.agentImage.tag }}
{{- .Values.agentImage.tag -}}
{{- else if eq .Chart.AppVersion "latest" }}
{{- print "latest" -}}
{{- else }}
{{- print "v" .Chart.AppVersion -}}
{{- end }}
{{- end -}}

{{- define "antreaControllerImageTag" -}}
{{- if .Values.controllerImage.tag }}
{{- .Values.controllerImage.tag -}}
{{- else if eq .Chart.AppVersion "latest" }}
{{- print "latest" -}}
{{- else }}
{{- print "v" .Chart.AppVersion -}}
{{- end }}
{{- end -}}

{{- define "antreaControllerImage" -}}
{{- if .Values.image }}
{{- print .Values.image.repository ":" (include "antreaImageTag" .) -}}
{{- else }}
{{- print .Values.controllerImage.repository ":" (include "antreaControllerImageTag" .) -}}
{{- end }}
{{- end -}}

{{- define "antreaAgentImage" -}}
{{- if .Values.image }}
{{- print .Values.image.repository ":" (include "antreaImageTag" .) -}}
{{- else }}
{{- print .Values.agentImage.repository ":" (include "antreaAgentImageTag" .) -}}
{{- end }}
{{- end -}}

{{- define "validateValues" -}}
Expand Down
16 changes: 8 additions & 8 deletions build/charts/antrea/templates/agent/daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ spec:
containers:
{{- end }}
- name: install-cni
image: {{ include "antreaImage" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
image: {{ include "antreaAgentImage" . | quote }}
imagePullPolicy: {{ .Values.agentImage.pullPolicy }}
resources: {{- .Values.agent.installCNI.resources | toYaml | nindent 12 }}
{{- if eq .Values.trafficEncapMode "networkPolicyOnly" }}
command: ["install_cni_chaining"]
Expand Down Expand Up @@ -127,8 +127,8 @@ spec:
containers:
{{- end }}
- name: antrea-agent
image: {{ include "antreaImage" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
image: {{ include "antreaAgentImage" . | quote }}
imagePullPolicy: {{ .Values.agentImage.pullPolicy }}
{{- if ((.Values.testing).coverage) }}
command: ["/bin/sh"]
args: ["-c", "sleep 2; antrea-agent-coverage -test.run=TestBincoverRunMain -test.coverprofile=antrea-agent.cov.out -args-file=/agent-arg-file; while true; do sleep 5 & wait $!; done"]
Expand Down Expand Up @@ -257,8 +257,8 @@ spec:
{{- toYaml . | trim | nindent 10 }}
{{- end }}
- name: antrea-ovs
image: {{ include "antreaImage" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
image: {{ include "antreaAgentImage" . | quote }}
imagePullPolicy: {{ .Values.agentImage.pullPolicy }}
resources: {{- .Values.agent.antreaOVS.resources | toYaml | nindent 12 }}
command: ["start_ovs"]
args:
Expand Down Expand Up @@ -313,8 +313,8 @@ spec:
subPath: openvswitch
{{- if eq .Values.trafficEncryptionMode "ipsec" }}
- name: antrea-ipsec
image: {{ include "antreaImage" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
image: {{ include "antreaAgentImage" . | quote }}
imagePullPolicy: {{ .Values.agentImage.pullPolicy }}
resources: {{- .Values.agent.antreaIPsec.resources | toYaml | nindent 12 }}
command: ["start_ovs_ipsec"]
livenessProbe:
Expand Down
4 changes: 2 additions & 2 deletions build/charts/antrea/templates/controller/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ spec:
serviceAccountName: antrea-controller
containers:
- name: antrea-controller
image: {{ include "antreaImage" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
image: {{ include "antreaControllerImage" . | quote }}
imagePullPolicy: {{ .Values.controllerImage.pullPolicy }}
resources: {{- .Values.controller.antreaController.resources | toYaml | nindent 12 }}
{{- if ((.Values.testing).coverage) }}
command: ["/bin/sh"]
Expand Down
12 changes: 10 additions & 2 deletions build/charts/antrea/values.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
# -- Container image to use for Antrea components.
image:
repository: "antrea/antrea-ubuntu"
# DEPRECATED: use agentImage and controllerImage instead.
image: {}
# -- Container image to use for the antrea-agent component.
agentImage:
repository: "antrea/antrea-agent-ubuntu"
pullPolicy: "IfNotPresent"
tag: ""
# -- Container image to use for the antrea-controller component.
controllerImage:
repository: "antrea/antrea-controller-ubuntu"
pullPolicy: "IfNotPresent"
tag: ""

Expand Down
47 changes: 47 additions & 0 deletions build/images/Dockerfile.agent.build.coverage
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Copyright 2024 Antrea Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ARG GO_VERSION
ARG BUILD_TAG
FROM golang:${GO_VERSION} as antrea-build

WORKDIR /antrea

COPY go.mod /antrea/go.mod

RUN go mod download

COPY . /antrea

RUN make antrea-agent antrea-cni antrea-agent-instr-binary
# Disable CGO for antctl in case it is copied outside of the container image. It
# also reduces the size of the binary and aligns with how we distribute antctl
# in release assets.
RUN CGO_ENABLED=0 make antctl-linux antctl-instr-binary
RUN mv bin/antctl-linux bin/antctl

FROM antrea/base-ubuntu:${BUILD_TAG}

LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
LABEL description="The Docker image to deploy the antrea-agent with code coverage measurement enabled (used for testing)."

USER root

COPY build/images/scripts/* /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antrea-agent /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antrea-agent-coverage /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antrea-cni /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antctl /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antctl-coverage /usr/local/bin/
COPY --from=antrea-build /antrea/test/e2e/coverage/agent-arg-file /
44 changes: 44 additions & 0 deletions build/images/Dockerfile.build.agent.ubuntu
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Copyright 2024 Antrea Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ARG GO_VERSION
ARG BUILD_TAG
FROM golang:${GO_VERSION} as antrea-build

WORKDIR /antrea

COPY go.mod /antrea/go.mod

RUN go mod download

COPY . /antrea

RUN make antrea-agent antrea-cni
# Disable CGO for antctl in case it is copied outside of the container image. It
# also reduces the size of the binary and aligns with how we distribute antctl
# in release assets.
RUN CGO_ENABLED=0 make antctl-linux
RUN mv bin/antctl-linux bin/antctl

FROM antrea/base-ubuntu:${BUILD_TAG}

LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
LABEL description="The Docker image to deploy the antrea-agent."

USER root

COPY build/images/scripts/* /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antrea-agent /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antrea-cni /usr/local/bin/
COPY --from=antrea-build /antrea/bin/antctl /usr/local/bin/
Loading

0 comments on commit fad0b8f

Please sign in to comment.