diff --git a/Makefile b/Makefile index 9dc908dd652..307d97b288e 100644 --- a/Makefile +++ b/Makefile @@ -419,9 +419,8 @@ build-migrator: manifest: @echo "===> Generating dev manifest for Antrea <===" $(CURDIR)/hack/generate-standard-manifests.sh --mode dev --out build/yamls - $(CURDIR)/hack/generate-manifest-windows.sh --mode dev > build/yamls/antrea-windows.yml - $(CURDIR)/hack/generate-manifest-windows.sh --mode dev --containerd > build/yamls/antrea-windows-containerd.yml - $(CURDIR)/hack/generate-manifest-windows.sh --mode dev --containerd --include-ovs > build/yamls/antrea-windows-containerd-with-ovs.yml + $(CURDIR)/hack/generate-manifest-windows.sh --mode dev > build/yamls/antrea-windows-containerd.yml + $(CURDIR)/hack/generate-manifest-windows.sh --mode dev --include-ovs > build/yamls/antrea-windows-containerd-with-ovs.yml $(CURDIR)/hack/update-checksum-windows.sh $(CURDIR)/hack/generate-manifest-flow-aggregator.sh --mode dev > build/yamls/flow-aggregator.yml diff --git a/build/yamls/antrea-windows.yml b/build/yamls/antrea-windows.yml deleted file mode 100644 index adc647323c1..00000000000 --- a/build/yamls/antrea-windows.yml +++ /dev/null @@ -1,300 +0,0 @@ -apiVersion: v1 -data: - Run-AntreaAgent.ps1: | - $ErrorActionPreference = "Stop" - # wins will rename the binary when executing it. So we need to copy the binary everytime before running it. - mkdir -force /host/k/antrea/bin - cp /k/antrea/bin/* /host/k/antrea/bin/ - C:/k/antrea/utils/wins.exe cli process run --path /k/antrea/bin/antrea-agent.exe --args "--config=/k/antrea/etc/antrea-agent.conf --logtostderr=false --log_dir=/var/log/antrea/ --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0" --envs "KUBERNETES_SERVICE_HOST=$env:KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT=$env:KUBERNETES_SERVICE_PORT ANTREA_SERVICE_HOST=$env:ANTREA_SERVICE_HOST ANTREA_SERVICE_PORT=$env:ANTREA_SERVICE_PORT NODE_NAME=$env:NODE_NAME KUBE_DNS_SERVICE_HOST=$env:KUBE_DNS_SERVICE_HOST KUBE_DNS_SERVICE_PORT=$env:KUBE_DNS_SERVICE_PORT" -kind: ConfigMap -metadata: - labels: - app: antrea - name: antrea-agent-windows - namespace: kube-system ---- -apiVersion: v1 -data: - antrea-agent.conf: | - # FeatureGates is a map of feature names to bools that enable or disable experimental features. - featureGates: - # Enable antrea proxy which provides ServiceLB for in-cluster services in antrea agent. - # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on - # Service traffic. Note that this feature gate is deprecated since this feature was - # promoted to GA in v1.14. - # AntreaProxy: true - - # Enable NodePortLocal feature to make the Pods reachable externally through NodePort - # NodePortLocal: true - - # Enable flowexporter which exports polled conntrack connections as IPFIX flow records from each agent to a configured collector. - # FlowExporter: false - - # Name of the OpenVSwitch bridge antrea-agent will create and use. - # Make sure it doesn't conflict with your existing OpenVSwitch bridges. - #ovsBridge: br-int - - # Name of the interface antrea-agent will create and use for host <--> pod communication. - # Make sure it doesn't conflict with your existing interfaces. - #hostGateway: antrea-gw0 - - # Encapsulation mode for communication between Pods across Nodes, supported values: - # - geneve (default) - # - vxlan - # - stt - #tunnelType: geneve - - # TunnelPort is the destination port for UDP and TCP based tunnel protocols - # (Geneve, VXLAN, and STT). If zero, it will use the assigned IANA port for the - # protocol, i.e. 6081 for Geneve, 4789 for VXLAN, and 7471 for STT. - #tunnelPort: 0 - - # Default MTU to use for the host gateway interface and the network interface of each Pod. - # If omitted, antrea-agent will discover the MTU of the Node's primary interface and - # also adjust MTU to accommodate for tunnel encapsulation overhead. - #defaultMTU: 1450 - - # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be - # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When - # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. - #serviceCIDR: 10.96.0.0/12 - - # The port for the antrea-agent APIServer to serve on. - #apiPort: 10350 - - # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. - #enablePrometheusMetrics: true - - # Provide the IPFIX collector address as a string with format :[][:]. - # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If - # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be - # wrapped with []. When the collector is running in-cluster as a Service, set - # to /. For example, - # "flow-aggregator/flow-aggregator" can be provided to connect to the Antrea - # Flow Aggregator Service. - # If PORT is empty, we default to 4739, the standard IPFIX port. - # If no PROTO is given, we consider "tls" as default. We support "tls", "tcp" and - # "udp" protocols. "tls" is used for securing communication between flow exporter and - # flow aggregator. - #flowCollectorAddr: "flow-aggregator/flow-aggregator:4739:tls" - - # Provide flow poll interval as a duration string. This determines how often the - # flow exporter dumps connections from the conntrack module. Flow poll interval - # should be greater than or equal to 1s (one second). - # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - #flowPollInterval: "5s" - - # Provide the active flow export timeout, which is the timeout after which a flow - # record is sent to the collector for active flows. Thus, for flows with a continuous - # stream of packets, a flow record will be exported to the collector once the elapsed - # time since the last export event is equal to the value of this timeout. - # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - #activeFlowExportTimeout: "30s" - - # Provide the idle flow export timeout, which is the timeout after which a flow - # record is sent to the collector for idle flows. A flow is considered idle if no - # packet matching this flow has been observed since the last export event. - # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - #idleFlowExportTimeout: "15s" - - # Enable TLS communication from flow exporter to flow aggregator. - #enableTLSToFlowAggregator: true - - # Determines how traffic is encapsulated. It has the following options: - # encap(default): Inter-node Pod traffic is always encapsulated and Pod to external network - # traffic is SNAT'd. - # noEncap: Inter-node Pod traffic is not encapsulated; Pod to external network traffic is - # SNAT'd if noSNAT is not set to true. Underlying network must be capable of - # supporting Pod traffic across IP subnets. - # hybrid: noEncap if source and destination Nodes are on the same subnet, otherwise encap. - # - #trafficEncapMode: encap - - # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes. - # If there are multiple IP addresses configured on the interface, the first one is used. The IP - # address used for tunneling or routing traffic to remote Nodes is decided in the following order of - # preference (from highest to lowest): - # 1. transportInterface - # 2. transportInterfaceCIDRs - # 3. The Node IP - #transportInterface: - - # The network CIDRs of the interface on Node which is used for tunneling or routing the traffic across - # Nodes. If there are multiple interfaces configured the same network CIDR, the first one is used. The - # IP address used for tunneling or routing traffic to remote Nodes is decided in the following order of - # preference (from highest to lowest): - # 1. transportInterface - # 2. transportInterfaceCIDRs - # 3. The Node IP - #transportInterfaceCIDRs: [,] - - # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. - # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. - #kubeAPIServerOverride: "" - - # Option antreaProxy contains AntreaProxy related configuration options. - antreaProxy: - # To disable AntreaProxy, set this to false. It should be enabled on Windows, otherwise NetworkPolicy will - # not take effect on Service traffic. - enable: true - # ProxyAll tells antrea-agent to proxy ClusterIP Service traffic, regardless of where they come from. - # Therefore, running kube-proxy is no longer required. This requires the AntreaProxy feature to be enabled. - # Note that this option is experimental. If kube-proxy is removed, option kubeAPIServerOverride must be used to access - # apiserver directly. - proxyAll: true - # The value of the "service.kubernetes.io/service-proxy-name" label for AntreaProxy to match. If it is set, - # then AntreaProxy will only handle Services with the label that equals the provided value. If it is not set, - # then AntreaProxy will only handle Services without the "service.kubernetes.io/service-proxy-name" label, - # but ignore Services with the label no matter what is the value. - serviceProxyName: "" - - nodePortLocal: - # Enable NodePortLocal, a feature used to make Pods reachable using port forwarding on the host. To - # enable this feature, you need to set "enable" to true, and ensure that the NodePortLocal feature - # gate is also enabled (which is the default). - # enable: false - # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port - # from that range will be assigned whenever a Pod's container defines a specific port to be exposed - # (each container can define a list of ports as pod.spec.containers[].ports), and all Node traffic - # directed to that port will be forwarded to the Pod. - # portRange: 40000-41000 - antrea-cni.conflist: | - { - "cniVersion":"0.3.0", - "name": "antrea", - "plugins": [ - { - "type": "antrea", - "ipam": { - "type": "host-local" - }, - "capabilities": {"dns": true} - } - ] - } -kind: ConfigMap -metadata: - labels: - app: antrea - name: antrea-windows-config - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: antrea - component: antrea-agent - name: antrea-agent-windows - namespace: kube-system -spec: - selector: - matchLabels: - app: antrea - component: antrea-agent - template: - metadata: - annotations: - checksum/agent-windows: 5af94f558d39950050ce9625ca7670bcca448b9ff3080a16902a5cae5d069210 - checksum/windows-config: 6ff4f8bd0b310ebe4d4612bdd9697ffb3d79e0e0eab3936420417dd5a8fc128d - labels: - app: antrea - component: antrea-agent - spec: - containers: - - args: - - -file - - /var/lib/antrea-windows/Run-AntreaAgent.ps1 - command: - - pwsh - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: antrea/antrea-windows:latest - imagePullPolicy: IfNotPresent - name: antrea-agent - volumeMounts: - - mountPath: \\.\pipe\rancher_wins - name: wins - - mountPath: /etc/antrea - name: antrea-windows-config - - mountPath: /var/lib/antrea-windows - name: antrea-agent-windows - - mountPath: /host/k/antrea/ - name: host-antrea-home - - mountPath: /var/log/antrea/ - name: var-log-antrea - hostNetwork: true - initContainers: - - args: - - -File - - /k/antrea/Install-WindowsCNI.ps1 - command: - - pwsh - image: antrea/antrea-windows:latest - imagePullPolicy: IfNotPresent - name: install-cni - volumeMounts: - - mountPath: /etc/antrea - name: antrea-windows-config - readOnly: true - - mountPath: /host/etc/cni/net.d - name: host-cni-conf - - mountPath: /host/opt/cni/bin - name: host-cni-bin - - mountPath: /host/k/antrea/ - name: host-antrea-home - - mountPath: /host/var/run/secrets/ - name: host-secrets-path - nodeSelector: - kubernetes.io/os: windows - priorityClassName: system-node-critical - serviceAccountName: antrea-agent - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - operator: Exists - volumes: - - configMap: - name: antrea-windows-config - name: antrea-windows-config - - configMap: - defaultMode: 420 - name: antrea-agent-windows - name: antrea-agent-windows - - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - name: host-cni-conf - - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - name: host-cni-bin - - hostPath: - path: /k/antrea - type: DirectoryOrCreate - name: host-antrea-home - - hostPath: - path: \\.\pipe\rancher_wins - name: wins - - hostPath: - path: /var/log/antrea/ - type: DirectoryOrCreate - name: var-log-antrea - - hostPath: - path: /var/run/secrets/ - type: DirectoryOrCreate - name: host-secrets-path - updateStrategy: - type: RollingUpdate diff --git a/build/yamls/windows/default/agent.yml b/build/yamls/windows/default/agent.yml deleted file mode 100644 index aa07d2e3367..00000000000 --- a/build/yamls/windows/default/agent.yml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - component: antrea-agent - name: antrea-agent-windows -spec: - selector: - matchLabels: - component: antrea-agent - template: - metadata: - labels: - component: antrea-agent - annotations: - checksum/windows-config: windows-config-checksum-placeholder - checksum/agent-windows: agent-windows-checksum-placeholder - spec: - containers: - - command: - - pwsh - args: - - -file - - /var/lib/antrea-windows/Run-AntreaAgent.ps1 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: antrea-windows - name: antrea-agent - volumeMounts: - - mountPath: \\.\pipe\rancher_wins - name: wins - - mountPath: /etc/antrea - name: antrea-windows-config - - mountPath: /var/lib/antrea-windows - name: antrea-agent-windows - - mountPath: /host/k/antrea/ - name: host-antrea-home - - mountPath: /var/log/antrea/ - name: var-log-antrea - hostNetwork: true - initContainers: - - command: - - pwsh - args: - - -File - - /k/antrea/Install-WindowsCNI.ps1 - image: antrea-windows - name: install-cni - volumeMounts: - - mountPath: /etc/antrea - name: antrea-windows-config - readOnly: true - - mountPath: /host/etc/cni/net.d - name: host-cni-conf - - mountPath: /host/opt/cni/bin - name: host-cni-bin - - mountPath: /host/k/antrea/ - name: host-antrea-home - - mountPath: /host/var/run/secrets/ - name: host-secrets-path - nodeSelector: - kubernetes.io/os: windows - priorityClassName: system-node-critical - serviceAccountName: antrea-agent - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - operator: Exists - volumes: - - configMap: - name: antrea-windows-config - name: antrea-windows-config - - configMap: - defaultMode: 420 - name: antrea-agent-windows - name: antrea-agent-windows - - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - name: host-cni-conf - - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - name: host-cni-bin - - hostPath: - path: /k/antrea - type: DirectoryOrCreate - name: host-antrea-home - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - name: var-log-antrea - hostPath: - path: /var/log/antrea/ - type: DirectoryOrCreate - - name: host-secrets-path - hostPath: - path: /var/run/secrets/ - type: DirectoryOrCreate - updateStrategy: - type: RollingUpdate diff --git a/build/yamls/windows/default/conf/Run-AntreaAgent.ps1 b/build/yamls/windows/default/conf/Run-AntreaAgent.ps1 deleted file mode 100644 index 948645e2e1b..00000000000 --- a/build/yamls/windows/default/conf/Run-AntreaAgent.ps1 +++ /dev/null @@ -1,5 +0,0 @@ -$ErrorActionPreference = "Stop" -# wins will rename the binary when executing it. So we need to copy the binary everytime before running it. -mkdir -force /host/k/antrea/bin -cp /k/antrea/bin/* /host/k/antrea/bin/ -C:/k/antrea/utils/wins.exe cli process run --path /k/antrea/bin/antrea-agent.exe --args "--config=/k/antrea/etc/antrea-agent.conf --logtostderr=false --log_dir=/var/log/antrea/ --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0" --envs "KUBERNETES_SERVICE_HOST=$env:KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT=$env:KUBERNETES_SERVICE_PORT ANTREA_SERVICE_HOST=$env:ANTREA_SERVICE_HOST ANTREA_SERVICE_PORT=$env:ANTREA_SERVICE_PORT NODE_NAME=$env:NODE_NAME KUBE_DNS_SERVICE_HOST=$env:KUBE_DNS_SERVICE_HOST KUBE_DNS_SERVICE_PORT=$env:KUBE_DNS_SERVICE_PORT" diff --git a/build/yamls/windows/default/kustomization.yml b/build/yamls/windows/default/kustomization.yml deleted file mode 100644 index 1d66ba94e78..00000000000 --- a/build/yamls/windows/default/kustomization.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- agent.yml -- ../base -namespace: kube-system -commonLabels: - app: antrea -configMapGenerator: -- files: - - conf/Run-AntreaAgent.ps1 - name: antrea-agent-windows -generatorOptions: - disableNameSuffixHash: true diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index 1456db960fc..6392dc78f92 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -375,171 +375,6 @@ function revert_snapshot_windows { sleep 5 } -function deliver_antrea_linux { - set -e - - echo "==== Start building and delivering Linux Docker images ====" - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull - if [[ "$TESTCASE" == "windows-networkpolicy-process" ]]; then - make windows-bin - fi - - echo "====== Delivering Antrea to all Nodes ======" - export_govc_env_var - - # Enable verbose log for troubleshooting. - sed -i "s/--v=0/--v=4/g" build/yamls/antrea-windows.yml - - if [[ "${PROXY_ALL}" == false && ${TESTCASE} =~ "windows-e2e" ]]; then - sed -i "s|.*proxyAll: true| proxyAll: false|g" build/yamls/antrea.yml build/yamls/antrea-windows.yml - else - echo "====== Updating yaml files to enable proxyAll ======" - KUBERNETES_SVC_EP_IP=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}') - KUBERNETES_SVC_EP_PORT=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].ports[0].port}') - KUBERNETES_SVC_EP_ADDR="${KUBERNETES_SVC_EP_IP}:${KUBERNETES_SVC_EP_PORT}" - sed -i "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBERNETES_SVC_EP_ADDR}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows.yml - fi - - cp -f build/yamls/*.yml $WORKDIR - docker save -o antrea-ubuntu.tar antrea/antrea-agent-ubuntu:latest antrea/antrea-controller-ubuntu:latest - - echo "===== Pull necessary images on Control-Plane node =====" - harbor_images=("agnhost:2.13" "nginx:1.15-alpine") - antrea_images=("e2eteam/agnhost:2.13" "docker.io/library/nginx:1.15-alpine") - common_images=("registry.k8s.io/e2e-test-images/agnhost:2.29") - for i in "${!harbor_images[@]}"; do - docker pull -q "${DOCKER_REGISTRY}/antrea/${harbor_images[i]}" - docker tag "${DOCKER_REGISTRY}/antrea/${harbor_images[i]}" "${antrea_images[i]}" - done - echo "===== Deliver Antrea to Linux worker nodes and pull necessary images on worker nodes =====" - kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 !~ role && $1 !~ /win/ {print $6}' | while read IP; do - rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" antrea-ubuntu.tar jenkins@${IP}:${WORKDIR}/antrea-ubuntu.tar - ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; ${PRINT_DOCKER_STATUS}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true - - for i in "${!harbor_images[@]}"; do - ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true - done - # Pull necessary images in advance to avoid transient error - for image in "${common_images[@]}"; do - ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "docker pull -q ${image}" || true - done - done - echo "==== Finish building and delivering Linux Docker images ====" -} - -function deliver_antrea_windows { - echo "===== Deliver Antrea Windows to Windows worker nodes and pull necessary images on Windows worker nodes =====" - rm -f antrea-windows.tar.gz - sed -i 's/if (!(Test-Path $AntreaAgentConfigPath))/if ($true)/' hack/windows/Helper.psm1 - kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 !~ role && $1 ~ /win/ {print $1}' | while read WORKER_NAME; do - revert_snapshot_windows ${WORKER_NAME} - - # Use a script to run antrea agent in windows Network Policy cases - if [ "$TESTCASE" == "windows-networkpolicy-process" ]; then - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "powershell stop-service kubelet" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "powershell stop-service docker" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "powershell rm C:\ProgramData\docker\docker.pid" || true - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "powershell start-service docker" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "powershell start-service kubelet" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "powershell start-service ovsdb-server" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "powershell start-service ovs-vswitchd" - echo "===== Use script to startup antrea agent =====" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "rm -rf /cygdrive/c/k/antrea && mkdir -p /cygdrive/c/k/antrea/bin && mkdir -p /cygdrive/c/k/antrea/etc && rm -rf /cygdrive/c/opt/cni/bin && mkdir -p /cygdrive/c/opt/cni/bin && mkdir -p /cygdrive/c/etc/cni/net.d" - scp -o StrictHostKeyChecking=no -T $KUBECONFIG Administrator@${IP}:/cygdrive/c/k/config - scp -o StrictHostKeyChecking=no -T bin/antrea-agent.exe Administrator@${IP}:/cygdrive/c/k/antrea/bin/ - scp -o StrictHostKeyChecking=no -T bin/antctl.exe Administrator@${IP}:/cygdrive/c/k/antrea/bin/antctl.exe - scp -o StrictHostKeyChecking=no -T bin/antrea-cni.exe Administrator@${IP}:/cygdrive/c/opt/cni/bin/antrea.exe - scp -o StrictHostKeyChecking=no -T hack/windows/Start-AntreaAgent.ps1 Administrator@${IP}:/cygdrive/c/k/antrea/ - scp -o StrictHostKeyChecking=no -T hack/windows/Stop-AntreaAgent.ps1 Administrator@${IP}:/cygdrive/c/k/antrea/ - scp -o StrictHostKeyChecking=no -T hack/windows/Helper.psm1 Administrator@${IP}:/cygdrive/c/k/antrea/ - scp -o StrictHostKeyChecking=no -T build/yamls/windows/base/conf/antrea-cni.conflist Administrator@${IP}:/cygdrive/c/etc/cni/net.d/10-antrea.conflist - scp -o StrictHostKeyChecking=no -T build/yamls/windows/base/conf/antrea-agent.conf Administrator@${IP}:/cygdrive/c/k/antrea/etc - else - if ! (test -f antrea-windows.tar.gz); then - # Compress antrea repo and copy it to a Windows node - mkdir -p jenkins - tar --exclude='./jenkins' -czf jenkins/antrea_repo.tar.gz -C "$(pwd)" . - for i in `seq 2`; do - timeout 2m scp -o StrictHostKeyChecking=no -T jenkins/antrea_repo.tar.gz Administrator@${IP}: && break - done - echo "=== Build Windows on Windows Node===" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull ${DOCKER_REGISTRY}/antrea/golang:${GO_VERSION}-nanoserver && docker tag ${DOCKER_REGISTRY}/antrea/golang:${GO_VERSION}-nanoserver golang:${GO_VERSION}-nanoserver" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "rm -rf antrea && mkdir antrea && cd antrea && tar -xzf ../antrea_repo.tar.gz > /dev/null && NO_PULL=${NO_PULL}; DOCKER_NETWORK=host make build-windows && docker save -o antrea-windows.tar antrea/antrea-windows:latest && gzip -f antrea-windows.tar" || true - for i in `seq 2`; do - timeout 2m scp -o StrictHostKeyChecking=no -T Administrator@${IP}:antrea/antrea-windows.tar.gz . && break - done - else - for i in `seq 2`; do - timeout 2m scp -o StrictHostKeyChecking=no -T antrea-windows.tar.gz Administrator@${IP}: && break - done - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker load -i antrea-windows.tar.gz" - fi - fi - # Some tests need us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13 image but it is not for windows/amd64 10.0.17763 - # Use e2eteam/agnhost:2.13 instead - harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2" "e2eteam-pause:3.2" "e2eteam-pause:3.2") - antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.13" "k8s.gcr.io/e2e-test-images/agnhost:2.29" "registry.k8s.io/e2e-test-images/agnhost:2.29" "e2eteam/jessie-dnsutils:1.0" "gcr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.0" "e2eteam/pause:3.2" "k8s.gcr.io/pause:3.2" "registry.k8s.io/pause:3.2") - common_images=("mcr.microsoft.com/windows/servercore/iis:latest") - # Pull necessary images in advance to avoid transient error - for i in "${!harbor_images[@]}"; do - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true - done - for image in "${common_images[@]}"; do - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull -q ${image}" || true - done - done - rm -f antrea-windows.tar.gz - echo "==== Finish building and delivering Windows Docker images ====" -} - -function build_and_deliver_antrea_windows_and_linux_docker_images { - echo "====== Cleanup Antrea Installation Before Delivering Antrea Windows and Antrea Linux Docker Images =====" - clean_antrea - kubectl delete -f ${WORKDIR}/antrea-windows.yml --ignore-not-found=true || true - kubectl delete -f ${WORKDIR}/kube-proxy-windows.yml --ignore-not-found=true || true - kubectl delete daemonset antrea-agent -n kube-system --ignore-not-found=true || true - kubectl delete -f ${WORKDIR}/antrea.yml --ignore-not-found=true || true - - prepare_env - ${CLEAN_STALE_IMAGES} - ${PRINT_DOCKER_STATUS} - chmod -R g-w build/images/ovs - chmod -R g-w build/images/base - - if [[ "$TESTCASE" == "windows-networkpolicy-process" ]]; then - make windows-bin - fi - - export_govc_env_var - - # Enable verbose log for troubleshooting. - sed -i "s/--v=0/--v=4/g" build/yamls/antrea-windows.yml - - if [[ "${PROXY_ALL}" == true ]]; then - echo "====== Updating yaml files to enable proxyAll ======" - KUBERNETES_SVC_EP_IP=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}') - KUBERNETES_SVC_EP_PORT=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].ports[0].port}') - KUBERNETES_SVC_EP_ADDR="${KUBERNETES_SVC_EP_IP}:${KUBERNETES_SVC_EP_PORT}" - sed -i "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBERNETES_SVC_EP_ADDR}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows.yml - sed -i "s|.*proxyAll: false| proxyAll: true|g" build/yamls/antrea.yml build/yamls/antrea-windows.yml - fi - - cp -f build/yamls/*.yml $WORKDIR - echo "====== Delivering Antrea to all Nodes ======" - set +e - deliver_antrea_windows &> deliver_antrea_windows.log & - deliver_antrea_windows_pid=$! - deliver_antrea_linux - linux_result=$? - wait $deliver_antrea_windows_pid - windows_result=$? - cat deliver_antrea_windows.log - if [ $windows_result -ne 0 ] || [ $linux_result -ne 0 ]; then - exit 1 - fi - set -e -} - function build_and_deliver_antrea_windows_and_linux_containerd_images { echo "====== Cleanup Antrea Installation Before Delivering Antrea Windows and Antrea Linux containerd Images =====" clean_antrea @@ -725,6 +560,14 @@ function deliver_antrea { fi done + if [[ "${PROXY_ALL}" == true ]]; then + echo "====== Updating yaml files to enable proxyAll ======" + KUBERNETES_SVC_EP_IP=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}') + KUBERNETES_SVC_EP_PORT=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].ports[0].port}') + KUBERNETES_SVC_EP_ADDR="${KUBERNETES_SVC_EP_IP}:${KUBERNETES_SVC_EP_PORT}" + sed -i "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBERNETES_SVC_EP_ADDR}\"|g" build/yamls/antrea.yml + sed -i "s|.*proxyAll: false| proxyAll: true|g" build/yamls/antrea.yml + fi echo "=== Append antrea-prometheus.yml to antrea.yml ===" echo "---" >> build/yamls/antrea.yml cat build/yamls/antrea-prometheus.yml >> build/yamls/antrea.yml @@ -892,41 +735,6 @@ function run_e2e_windows { tar -zcf antrea-test-logs.tar.gz antrea-test-logs } -function run_conformance_windows { - echo "====== Running Antrea Conformance Tests ======" - export GO111MODULE=on - export GOPATH=${WORKDIR}/go - export GOROOT=${GOLANG_RELEASE_DIR}/go - export GOCACHE=${WORKDIR}/.cache/go-build - export PATH=$GOROOT/bin:$PATH - - if [[ "$TESTCASE" == "windows-networkpolicy-process" ]]; then - # Antrea Windows agents are deployed with scripts as processes on host for Windows NetworkPolicy test - wait_for_antrea_windows_processes_ready - else - # Antrea Windows agent Pods are deployed for Windows Conformance test - clean_for_windows_install_cni - wait_for_antrea_windows_pods_ready - fi - - echo "====== Run test with conformance test ======" - export KUBE_TEST_REPO_LIST=${WORKDIR}/repo_list - if [ "$TESTCASE" == "windows-networkpolicy" ]; then - # Allow LinuxOnly mark in windows-networkpolicy because Antrea Windows supports NP functions. - ginkgo --noColor $E2ETEST_PATH -- --provider=skeleton --ginkgo.focus="$WINDOWS_NETWORKPOLICY_FOCUS" --ginkgo.skip="$WINDOWS_NETWORKPOLICY_SKIP" > windows_conformance_result_no_color.txt || true - else - ginkgo --noColor $E2ETEST_PATH -- --provider=skeleton --node-os-distro=windows --ginkgo.focus="$WINDOWS_CONFORMANCE_FOCUS" --ginkgo.skip="$WINDOWS_CONFORMANCE_SKIP" > windows_conformance_result_no_color.txt || true - fi - - if grep -Fxq "Test Suite Failed" windows_conformance_result_no_color.txt; then - echo "=== Failed cases exist ===" - TEST_FAILURE=true - collect_windows_network_info_and_logs - else - echo "All tests passed." - fi -} - function run_conformance_windows_containerd { echo "====== Running Antrea Conformance Tests ======" export GO111MODULE=on @@ -1202,13 +1010,6 @@ if [[ ${TESTCASE} =~ "windows" ]]; then else run_conformance_windows_containerd fi - else - build_and_deliver_antrea_windows_and_linux_docker_images - if [[ ${TESTCASE} =~ "e2e" ]]; then - run_e2e_windows - else - run_conformance_windows - fi fi elif [[ ${TESTCASE} =~ "e2e" ]]; then deliver_antrea diff --git a/hack/generate-manifest-windows.sh b/hack/generate-manifest-windows.sh index 1512f0d3a15..e6e13d600a3 100755 --- a/hack/generate-manifest-windows.sh +++ b/hack/generate-manifest-windows.sh @@ -25,7 +25,6 @@ Generate a YAML manifest to run Antrea on Windows Nodes, using Kustomize, and pr --mode (dev|release) Choose the configuration variant that you need (default is 'dev') --keep Debug flag which will preserve the generated kustomization.yml --help, -h Print this message and exit - --containerd Support for containerd runtime. --include-ovs Run Windows OVS processes inside antrea-ovs container in antrea-agent pod on Windows host with containerd runtime. @@ -44,7 +43,6 @@ function print_help { echoerr "Try '$0 --help' for more information." } -RUNTIME="" MODE="dev" KEEP=false INCLUDE_OVS=false @@ -62,10 +60,6 @@ case $key in KEEP=true shift ;; - --containerd) - RUNTIME="containerd" - shift - ;; --include-ovs) INCLUDE_OVS=true shift @@ -117,13 +111,9 @@ TMP_DIR=$(mktemp -d $KUSTOMIZATION_DIR/overlays.XXXXXXXX) pushd $TMP_DIR > /dev/null -BASE=../../default -if [ "$RUNTIME" == "containerd" ]; then - if $INCLUDE_OVS; then - BASE=../../containerd-with-ovs - else - BASE=../../containerd - fi +BASE=../../containerd +if $INCLUDE_OVS; then + BASE=../../containerd-with-ovs fi mkdir $MODE && cd $MODE diff --git a/hack/release/prepare-assets.sh b/hack/release/prepare-assets.sh index 66e1fa42faf..d1d389a80c8 100755 --- a/hack/release/prepare-assets.sh +++ b/hack/release/prepare-assets.sh @@ -110,9 +110,8 @@ export CONTROLLER_IMG_NAME=antrea/antrea-controller-ubuntu ./hack/generate-standard-manifests.sh --mode release --out "$OUTPUT_DIR" export IMG_NAME=antrea/antrea-windows -./hack/generate-manifest-windows.sh --mode release > "$OUTPUT_DIR"/antrea-windows.yml -./hack/generate-manifest-windows.sh --mode release --containerd > "$OUTPUT_DIR"/antrea-windows-containerd.yml -./hack/generate-manifest-windows.sh --mode release --containerd --include-ovs > "$OUTPUT_DIR"/antrea-windows-containerd-with-ovs.yml +./hack/generate-manifest-windows.sh --mode release > "$OUTPUT_DIR"/antrea-windows-containerd.yml +./hack/generate-manifest-windows.sh --mode release --include-ovs > "$OUTPUT_DIR"/antrea-windows-containerd-with-ovs.yml export IMG_NAME=antrea/flow-aggregator ./hack/generate-manifest-flow-aggregator.sh --mode release > "$OUTPUT_DIR"/flow-aggregator.yml diff --git a/hack/update-checksum-windows.sh b/hack/update-checksum-windows.sh index 43c8650671b..f418b4d2c20 100755 --- a/hack/update-checksum-windows.sh +++ b/hack/update-checksum-windows.sh @@ -21,7 +21,6 @@ YAMLS_DIR="${WORK_DIR}"/../build/yamls MANIFESTS=$(ls $YAMLS_DIR/antrea-windows*.yml) WINDOWS_DIR="${YAMLS_DIR}"/windows BASE_CONF_FILES="${WINDOWS_DIR}/base/conf/antrea-agent.conf ${WINDOWS_DIR}/base/conf/antrea-cni.conflist" -DEFAULT_CONF_FILES="${WINDOWS_DIR}/default/conf/Run-AntreaAgent.ps1" CONTAINERD_CONF_FILES="${WINDOWS_DIR}/containerd/conf/Install-WindowsCNI-Containerd.ps1 \ ${WINDOWS_DIR}/containerd/conf/Run-AntreaAgent-Containerd.ps1" CONTAINERD_WITH_OVS_CONF_FILES="${WINDOWS_DIR}/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1 \ @@ -29,8 +28,6 @@ CONTAINERD_WITH_OVS_CONF_FILES="${WINDOWS_DIR}/containerd-with-ovs/conf/Run-Antr checksum_windows_config=$(cat ${BASE_CONF_FILES} | sha256sum | cut -d " " -f 1) -checksum_default=$(cat ${DEFAULT_CONF_FILES} | sha256sum | cut -d " " -f 1) - checksum_containerd=$( cat ${CONTAINERD_CONF_FILES} | sha256sum | cut -d " " -f 1) checksum_containerd_with_ovs=$(cat ${CONTAINERD_CONF_FILES} ${CONTAINERD_WITH_OVS_CONF_FILES} | sha256sum | cut -d " " -f 1) @@ -39,6 +36,5 @@ for file in ${MANIFESTS[@]}; do sed -i.bak "s/windows-config-checksum-placeholder/${checksum_windows_config}/g" ${file} done -sed -i.bak "s/agent-windows-checksum-placeholder/${checksum_default}/g" ${YAMLS_DIR}/antrea-windows.yml sed -i.bak "s/agent-windows-checksum-placeholder/${checksum_containerd}/g" ${YAMLS_DIR}/antrea-windows-containerd.yml sed -i.bak "s/agent-windows-checksum-placeholder/${checksum_containerd_with_ovs}/g" ${YAMLS_DIR}/antrea-windows-containerd-with-ovs.yml diff --git a/pkg/agent/cniserver/interface_configuration_linux.go b/pkg/agent/cniserver/interface_configuration_linux.go index da83ff6569f..822f6dcc9dd 100644 --- a/pkg/agent/cniserver/interface_configuration_linux.go +++ b/pkg/agent/cniserver/interface_configuration_linux.go @@ -647,7 +647,3 @@ func isVeth(link netlink.Link) bool { _, isVeth := link.(*netlink.Veth) return isVeth } - -func getOVSInterfaceType(ovsPortName string) int { - return defaultOVSInterfaceType -} diff --git a/pkg/agent/cniserver/interface_configuration_windows.go b/pkg/agent/cniserver/interface_configuration_windows.go index a083354369d..72603777589 100644 --- a/pkg/agent/cniserver/interface_configuration_windows.go +++ b/pkg/agent/cniserver/interface_configuration_windows.go @@ -48,10 +48,7 @@ var ( hostInterfaceExistsFunc = util.HostInterfaceExists getNetInterfaceAddrsFunc = getNetInterfaceAddrs createHnsEndpointFunc = createHnsEndpoint - getNamespaceEndpointIDsFunc = hcn.GetNamespaceEndpointIds - hotAttachEndpointFunc = hcsshim.HotAttachEndpoint attachEndpointInNamespaceFunc = attachEndpointInNamespace - isContainerAttachOnEndpointFunc = isContainerAttachOnEndpoint getHcnEndpointByIDFunc = hcn.GetEndpointByID deleteHnsEndpointFunc = deleteHnsEndpoint removeEndpointFromNamespaceFunc = hcn.RemoveNamespaceEndpoint @@ -221,49 +218,17 @@ func (ic *ifConfigurator) createContainerLink(endpointName string, result *curre // attachContainerLink takes the result of the IPAM plugin, and adds the appropriate IP // addresses and routes to the interface. // For different CRI runtimes we need to use the appropriate Windows container API: -// - Docker runtime: HNS API // - containerd runtime: HCS API func attachContainerLink(ep *hcsshim.HNSEndpoint, containerID, sandbox, containerIFDev string) (*current.Interface, error) { - var attached bool var err error var hcnEp *hcn.HostComputeEndpoint - if isDockerContainer(sandbox) { - // Docker runtime - attached, err = isContainerAttachOnEndpointFunc(ep, containerID) - if err != nil { - return nil, err - } - } else { - // containerd runtime - if hcnEp, err = getHcnEndpointByIDFunc(ep.Id); err != nil { - return nil, err - } - attachedEpIds, err := getNamespaceEndpointIDsFunc(sandbox) - if err != nil { - return nil, err - } - for _, existingEP := range attachedEpIds { - if existingEP == hcnEp.Id { - attached = true - break - } - } + + if hcnEp, err = getHcnEndpointByIDFunc(ep.Id); err != nil { + return nil, err } - if attached { - klog.V(2).Infof("HNS Endpoint %s already attached on container %s", ep.Id, containerID) - } else { - if hcnEp == nil { - // Docker runtime - if err := hotAttachEndpointFunc(containerID, ep.Id); err != nil { - return nil, err - } - } else { - // containerd runtime - if err := attachEndpointInNamespaceFunc(hcnEp, sandbox); err != nil { - return nil, err - } - } + if err := attachEndpointInNamespaceFunc(hcnEp, sandbox); err != nil { + return nil, err } containerIface := ¤t.Interface{ Name: containerIFDev, @@ -273,10 +238,6 @@ func attachContainerLink(ep *hcsshim.HNSEndpoint, containerID, sandbox, containe return containerIface, nil } -func isContainerAttachOnEndpoint(endpoint *hcsshim.HNSEndpoint, containerID string) (bool, error) { - return endpoint.IsAttached(containerID) -} - func attachEndpointInNamespace(hcnEp *hcn.HostComputeEndpoint, sandbox string) error { return hcnEp.NamespaceAttach(sandbox) } @@ -485,15 +446,6 @@ func (ic *ifConfigurator) getInterceptedInterfaces( return nil, nil, errors.New("getInterceptedInterfaces is unsupported on Windows") } -// getOVSInterfaceType returns "internal". Windows uses internal OVS interface for container vNIC. -func getOVSInterfaceType(ovsPortName string) int { - ifaceName := fmt.Sprintf("vEthernet (%s)", ovsPortName) - if !hostInterfaceExistsFunc(ifaceName) { - return defaultOVSInterfaceType - } - return internalOVSInterfaceType -} - func (ic *ifConfigurator) addPostInterfaceCreateHook(containerID, endpointName string, containerAccess *containerAccessArbitrator, hook postInterfaceCreateHook) error { if containerAccess == nil { return fmt.Errorf("container lock cannot be null") diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index 351b13c19bd..d65533da3e1 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -55,9 +55,6 @@ const ( ) const ( - defaultOVSInterfaceType int = iota //nolint suppress deadcode check for windows - internalOVSInterfaceType - defaultIFDevName = "eth0" ) @@ -265,15 +262,11 @@ func (pc *podConfigurator) configureInterfacesCommon( func (pc *podConfigurator) createOVSPort(ovsPortName string, ovsAttachInfo map[string]interface{}, vlanID uint16) (string, error) { var portUUID string var err error - switch getOVSInterfaceType(ovsPortName) { - case internalOVSInterfaceType: - portUUID, err = pc.ovsBridgeClient.CreateInternalPort(ovsPortName, 0, "", ovsAttachInfo) - default: - if vlanID == 0 { - portUUID, err = pc.ovsBridgeClient.CreatePort(ovsPortName, ovsPortName, ovsAttachInfo) - } else { - portUUID, err = pc.ovsBridgeClient.CreateAccessPort(ovsPortName, ovsPortName, ovsAttachInfo, vlanID) - } + + if vlanID == 0 { + portUUID, err = pc.ovsBridgeClient.CreatePort(ovsPortName, ovsPortName, ovsAttachInfo) + } else { + portUUID, err = pc.ovsBridgeClient.CreateAccessPort(ovsPortName, ovsPortName, ovsAttachInfo, vlanID) } if err != nil { klog.Errorf("Failed to add OVS port %s, remove from local cache: %v", ovsPortName, err) @@ -493,55 +486,6 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain return nil } -func (pc *podConfigurator) connectInterfaceToOVSCommon(ovsPortName, netNS string, containerConfig *interfacestore.InterfaceConfig) error { - // create OVS Port and add attach container configuration into external_ids - containerID := containerConfig.ContainerID - klog.V(2).Infof("Adding OVS port %s for container %s", ovsPortName, containerID) - ovsAttachInfo := BuildOVSPortExternalIDs(containerConfig) - portUUID, err := pc.createOVSPort(ovsPortName, ovsAttachInfo, containerConfig.VLANID) - if err != nil { - return fmt.Errorf("failed to add OVS port for container %s: %v", containerID, err) - } - // Remove OVS port if any failure occurs in later manipulation. - defer func() { - if err != nil { - _ = pc.ovsBridgeClient.DeletePort(portUUID) - } - }() - - var ofPort int32 - // Not needed for a secondary network interface. - if !pc.isSecondaryNetwork { - // GetOFPort will wait for up to 1 second for OVSDB to report the OFPort number. - ofPort, err = pc.ovsBridgeClient.GetOFPort(ovsPortName, false) - if err != nil { - return fmt.Errorf("failed to get of_port of OVS port %s: %v", ovsPortName, err) - } - klog.V(2).InfoS("Setting up Openflow entries for Pod interface", "container", containerID, "port", ovsPortName) - if err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IPs, containerConfig.MAC, uint32(ofPort), containerConfig.VLANID, nil); err != nil { - return fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err) - } - } - - containerConfig.OVSPortConfig = &interfacestore.OVSPortConfig{PortUUID: portUUID, OFPort: ofPort} - // Add containerConfig into local cache - pc.ifaceStore.AddInterface(containerConfig) - - // Not needed for a secondary network interface. - if !pc.isSecondaryNetwork { - // Notify the Pod update event to required components. - event := agenttypes.PodUpdate{ - PodName: containerConfig.PodName, - PodNamespace: containerConfig.PodNamespace, - ContainerID: containerConfig.ContainerID, - NetNS: netNS, - IsAdd: true, - } - pc.podUpdateNotifier.Notify(event) - } - return nil -} - // disconnectInterfaceFromOVS disconnects an existing interface from ovs br-int. func (pc *podConfigurator) disconnectInterfaceFromOVS(containerConfig *interfacestore.InterfaceConfig) error { containerID := containerConfig.ContainerID diff --git a/pkg/agent/cniserver/pod_configuration_linux.go b/pkg/agent/cniserver/pod_configuration_linux.go index 75dd9fd6d5e..fe281f06330 100644 --- a/pkg/agent/cniserver/pod_configuration_linux.go +++ b/pkg/agent/cniserver/pod_configuration_linux.go @@ -18,11 +18,14 @@ package cniserver import ( + "fmt" + current "github.com/containernetworking/cni/pkg/types/100" "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/cniserver/ipam" "antrea.io/antrea/pkg/agent/interfacestore" + agenttypes "antrea.io/antrea/pkg/agent/types" ) // connectInterfaceToOVS connects an existing interface to the OVS bridge. @@ -38,6 +41,55 @@ func (pc *podConfigurator) connectInterfaceToOVS( return containerConfig, pc.connectInterfaceToOVSCommon(ovsPortName, netNS, containerConfig) } +func (pc *podConfigurator) connectInterfaceToOVSCommon(ovsPortName, netNS string, containerConfig *interfacestore.InterfaceConfig) error { + // create OVS Port and add attach container configuration into external_ids + containerID := containerConfig.ContainerID + klog.V(2).Infof("Adding OVS port %s for container %s", ovsPortName, containerID) + ovsAttachInfo := BuildOVSPortExternalIDs(containerConfig) + portUUID, err := pc.createOVSPort(ovsPortName, ovsAttachInfo, containerConfig.VLANID) + if err != nil { + return fmt.Errorf("failed to add OVS port for container %s: %v", containerID, err) + } + // Remove OVS port if any failure occurs in later manipulation. + defer func() { + if err != nil { + _ = pc.ovsBridgeClient.DeletePort(portUUID) + } + }() + + var ofPort int32 + // Not needed for a secondary network interface. + if !pc.isSecondaryNetwork { + // GetOFPort will wait for up to 1 second for OVSDB to report the OFPort number. + ofPort, err = pc.ovsBridgeClient.GetOFPort(ovsPortName, false) + if err != nil { + return fmt.Errorf("failed to get of_port of OVS port %s: %v", ovsPortName, err) + } + klog.V(2).InfoS("Setting up Openflow entries for Pod interface", "container", containerID, "port", ovsPortName) + if err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IPs, containerConfig.MAC, uint32(ofPort), containerConfig.VLANID, nil); err != nil { + return fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err) + } + } + + containerConfig.OVSPortConfig = &interfacestore.OVSPortConfig{PortUUID: portUUID, OFPort: ofPort} + // Add containerConfig into local cache + pc.ifaceStore.AddInterface(containerConfig) + + // Not needed for a secondary network interface. + if !pc.isSecondaryNetwork { + // Notify the Pod update event to required components. + event := agenttypes.PodUpdate{ + PodName: containerConfig.PodName, + PodNamespace: containerConfig.PodNamespace, + ContainerID: containerConfig.ContainerID, + NetNS: netNS, + IsAdd: true, + } + pc.podUpdateNotifier.Notify(event) + } + return nil +} + func (pc *podConfigurator) configureInterfaces( podName, podNamespace, containerID, containerNetNS string, containerIFDev string, mtu int, sriovVFDeviceID string, diff --git a/pkg/agent/cniserver/pod_configuration_linux_test.go b/pkg/agent/cniserver/pod_configuration_linux_test.go index cc78e068c09..4f51bd4892b 100644 --- a/pkg/agent/cniserver/pod_configuration_linux_test.go +++ b/pkg/agent/cniserver/pod_configuration_linux_test.go @@ -46,7 +46,7 @@ type fakeInterfaceConfigurator struct { configureContainerLinkError error removeContainerLinkError error advertiseContainerAddrError error - ovsInterfaceTypeMapping map[string]int + ovsInterfaceTypeMapping string validateVFRepInterfaceError error validateContainerPeerInterfaceError error containerVethPair *vethPair @@ -248,7 +248,6 @@ func TestCreateOVSPort(t *testing.T) { for _, tc := range []struct { name string portName string - portType int vlanID uint16 createOVSPort bool createOVSAccessPort bool @@ -256,19 +255,17 @@ func TestCreateOVSPort(t *testing.T) { { name: "create-general-port", portName: "p1", - portType: defaultOVSInterfaceType, vlanID: 0, createOVSPort: true, }, { name: "create-access-port", portName: "p3", - portType: defaultOVSInterfaceType, vlanID: 10, createOVSAccessPort: true, }, } { t.Run(tc.name, func(t *testing.T) { - testIfaceConfigurator := &fakeInterfaceConfigurator{ovsInterfaceTypeMapping: map[string]int{tc.portName: tc.portType}} + testIfaceConfigurator := &fakeInterfaceConfigurator{ovsInterfaceTypeMapping: tc.portName} podConfigurator := createPodConfigurator(controller, testIfaceConfigurator) containerConfig := buildContainerConfig(tc.portName, containerID, podName, podNamespace, ¤t.Interface{Mac: "01:02:03:04:05:06"}, ipamResult.IPs, tc.vlanID) attachInfo := BuildOVSPortExternalIDs(containerConfig) diff --git a/pkg/agent/cniserver/pod_configuration_windows.go b/pkg/agent/cniserver/pod_configuration_windows.go index 8842a6bf416..44734e4f20e 100644 --- a/pkg/agent/cniserver/pod_configuration_windows.go +++ b/pkg/agent/cniserver/pod_configuration_windows.go @@ -26,7 +26,6 @@ import ( "antrea.io/antrea/pkg/agent/cniserver/ipam" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/types" - "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/util/k8s" ) @@ -72,20 +71,10 @@ func (pc *podConfigurator) connectInterfaceToOVS( // Use the outer veth interface name as the OVS port name. ovsPortName := hostIface.Name containerConfig := buildContainerConfig(ovsPortName, containerID, podName, podNamespace, containerIface, ips, vlanID) - hostIfAlias := util.VirtualAdapterName(ovsPortName) - // - For containerd runtime, the container interface is created after CNI replying the network setup result. - // So for such case we need to use asynchronous way to wait for interface to be created: we create the OVS port - // and set the OVS Interface type "" first, and change the OVS Interface type to "internal" to connect to the - // container interface after it is created. After OVS connects to the container interface, an OFPort is allocated. - // - For Docker runtime, the container interface is created after antrea-agent attaches the HNSEndpoint to the - // sandbox container, so we create OVS port synchronously. - // - Here antrea-agent determines the way of OVS port creation by checking if container interface is yet created. - // If one day containerd runtime changes the behavior and container interface can be created when attaching - // HNSEndpoint/HostComputeEndpoint, the current implementation will still work. It will choose the synchronized - // way to create OVS port. - if hostInterfaceExistsFunc(hostIfAlias) { - return containerConfig, pc.connectInterfaceToOVSCommon(ovsPortName, netNS, containerConfig) - } + // The container interface is created after the CNI returns the network setup result. + // Because of this, we need to wait asynchronously for the interface to be created: we create the OVS port + // and set the OVS Interface type "" first, and change the OVS Interface type to "internal" to connect to the + // container interface after it is created. After OVS connects to the container interface, an OFPort is allocated. klog.V(2).Infof("Adding OVS port %s for container %s", ovsPortName, containerID) ovsAttachInfo := BuildOVSPortExternalIDs(containerConfig) portUUID, err := pc.createOVSPort(ovsPortName, ovsAttachInfo, containerConfig.VLANID) diff --git a/pkg/agent/cniserver/server.go b/pkg/agent/cniserver/server.go index cec67536c14..f99f0d765e2 100644 --- a/pkg/agent/cniserver/server.go +++ b/pkg/agent/cniserver/server.go @@ -442,6 +442,9 @@ func (s *CNIServer) CmdAdd(ctx context.Context, request *cnipb.CniCmdRequest) (* result := &ipam.IPAMResult{Result: current.Result{CNIVersion: current.ImplementedSpecVersion}} netNS := s.hostNetNsPath(cniConfig.Netns) + if err := validateRuntime(netNS); err != nil { + return nil, fmt.Errorf("failed to validate container runtime for CmdAdd request: %w", err) + } isInfraContainer := isInfraContainer(netNS) success := false @@ -563,6 +566,11 @@ func (s *CNIServer) CmdDel(ctx context.Context, request *cnipb.CniCmdRequest) (* return response, nil } + netNS := s.hostNetNsPath(cniConfig.Netns) + if err := validateRuntime(netNS); err != nil { + return nil, fmt.Errorf("failed to validate container runtime for CmdDel request: %w", err) + } + return s.cmdDel(ctx, cniConfig) } @@ -575,6 +583,11 @@ func (s *CNIServer) CmdCheck(_ context.Context, request *cnipb.CniCmdRequest) ( return response, nil } + netNS := s.hostNetNsPath(cniConfig.Netns) + if err := validateRuntime(netNS); err != nil { + return nil, fmt.Errorf("failed to validate container runtime for CmdCheck request: %w", err) + } + infraContainer := cniConfig.getInfraContainer() s.containerAccess.lockContainer(infraContainer) defer s.containerAccess.unlockContainer(infraContainer) diff --git a/pkg/agent/cniserver/server_linux.go b/pkg/agent/cniserver/server_linux.go index d9f380608ea..0e21a557940 100644 --- a/pkg/agent/cniserver/server_linux.go +++ b/pkg/agent/cniserver/server_linux.go @@ -37,6 +37,12 @@ func isInfraContainer(netNS string) bool { return true } +// validateRuntime returns nil if the container runtime is supported by Antrea. +// Always return nil on Linux platform, because all container runtimes are supported. +func validateRuntime(netNS string) error { + return nil +} + // getInfraContainer returns the sandbox container ID of a Pod. // On Linux, it's always the ContainerID in the request. func (c *CNIConfig) getInfraContainer() string { diff --git a/pkg/agent/cniserver/server_windows.go b/pkg/agent/cniserver/server_windows.go index e1158ce9883..794c10e1d66 100644 --- a/pkg/agent/cniserver/server_windows.go +++ b/pkg/agent/cniserver/server_windows.go @@ -18,6 +18,7 @@ package cniserver import ( + "fmt" "strings" current "github.com/containernetworking/cni/pkg/types/100" @@ -64,6 +65,15 @@ func isDockerContainer(netNS string) bool { return netNS == dockerInfraContainerNetNS || strings.Contains(netNS, ":") } +// validateRuntime returns error if a container is created by Docker with the provided network namespace +// because the Docker support has been removed since Antrea 2.0. +func validateRuntime(netNS string) error { + if isDockerContainer(netNS) { + return fmt.Errorf("Docker runtime is not supported after Antrea 2.0 for Windows Nodes") + } + return nil +} + func getInfraContainer(containerID, netNS string) string { if isInfraContainer(netNS) { return containerID diff --git a/pkg/agent/cniserver/server_windows_test.go b/pkg/agent/cniserver/server_windows_test.go index ba562bae762..d8b80a13309 100644 --- a/pkg/agent/cniserver/server_windows_test.go +++ b/pkg/agent/cniserver/server_windows_test.go @@ -204,25 +204,6 @@ func (t *hnsTestUtil) createHnsEndpoint(request *hcsshim.HNSEndpoint) (*hcsshim. return request, t.hnsEndpointCreatErr } -func (t *hnsTestUtil) getNamespaceEndpointIDs(namespace string) ([]string, error) { - if t.isAttached { - t.addHostInterface() - return []string{t.endpointID}, nil - } - return []string{}, nil -} - -func (t *hnsTestUtil) hotAttachEndpoint(containerID string, epID string) error { - if t.endpointAttachErr == nil { - hostIfaces.Store(t.hostIfaceName, false) - } - return t.endpointAttachErr -} - -func (t *hnsTestUtil) isContainerAttachOnEndpoint(ep *hcsshim.HNSEndpoint, containerID string) (bool, error) { - return t.isAttached, nil -} - func (t *hnsTestUtil) getHcnEndpointByID(epID string) (*hcn.HostComputeEndpoint, error) { return t.hcnEndpoint, nil } @@ -246,10 +227,7 @@ func (t *hnsTestUtil) removeEndpointFromNamespace(namespace string, epID string) func (t *hnsTestUtil) setFunctions() { listHnsEndpointFunc = t.listHnsEndpointFunc createHnsEndpointFunc = t.createHnsEndpoint - getNamespaceEndpointIDsFunc = t.getNamespaceEndpointIDs - hotAttachEndpointFunc = t.hotAttachEndpoint attachEndpointInNamespaceFunc = t.attachEndpointInNamespace - isContainerAttachOnEndpointFunc = t.isContainerAttachOnEndpoint getHcnEndpointByIDFunc = t.getHcnEndpointByID deleteHnsEndpointFunc = t.deleteHnsEndpoint removeEndpointFromNamespaceFunc = t.removeEndpointFromNamespace @@ -258,10 +236,7 @@ func (t *hnsTestUtil) setFunctions() { func (t *hnsTestUtil) restore() { listHnsEndpointFunc = hcsshim.HNSListEndpointRequest createHnsEndpointFunc = createHnsEndpoint - getNamespaceEndpointIDsFunc = hcn.GetNamespaceEndpointIds - hotAttachEndpointFunc = hcsshim.HotAttachEndpoint attachEndpointInNamespaceFunc = attachEndpointInNamespace - isContainerAttachOnEndpointFunc = isContainerAttachOnEndpoint getHcnEndpointByIDFunc = hcn.GetEndpointByID deleteHnsEndpointFunc = deleteHnsEndpoint removeEndpointFromNamespaceFunc = hcn.RemoveNamespaceEndpoint @@ -321,9 +296,6 @@ func TestCmdAdd(t *testing.T) { oriIPAMResult := &ipam.IPAMResult{Result: *ipamResult} ctx := context.TODO() - dockerInfraContainer := "261a1970-5b6c-11ed-8caf-000c294e5d03" - dockerWorkContainer := "261e579a-5b6c-11ed-8caf-000c294e5d03" - unknownInfraContainer := generateUUID() containerdInfraContainer := generateUUID() defer mockHostInterfaceExists()() @@ -352,97 +324,6 @@ func TestCmdAdd(t *testing.T) { expectedErr error }{ { - name: "docker-infra-create-failure", - podName: "pod0", - containerID: dockerInfraContainer, - infraContainerID: dockerInfraContainer, - netns: "none", - ipamAdd: true, - ipamDel: true, - hnsEndpointCreateErr: fmt.Errorf("unable to create HnsEndpoint"), - errResponse: &cnipb.CniCmdResponse{ - Error: &cnipb.Error{ - Code: cnipb.ErrorCode_CONFIG_INTERFACE_FAILURE, - Message: "unable to create HnsEndpoint", - }, - }, - }, { - name: "docker-infra-attach-failure", - podName: "pod1", - containerID: dockerInfraContainer, - infraContainerID: dockerInfraContainer, - netns: "none", - ipamAdd: true, - ipamDel: true, - endpointAttachErr: fmt.Errorf("unable to attach HnsEndpoint"), - errResponse: &cnipb.CniCmdResponse{ - Error: &cnipb.Error{ - Code: cnipb.ErrorCode_CONFIG_INTERFACE_FAILURE, - Message: "failed to configure container IP: unable to attach HnsEndpoint", - }, - }, - }, { - name: "docker-infra-success", - podName: "pod2", - containerID: dockerInfraContainer, - infraContainerID: dockerInfraContainer, - netns: "none", - ipamAdd: true, - connectOVS: true, - containerIfaceExist: true, - }, { - name: "docker-workload-allocate-ip-failure", - podName: "pod3", - containerID: dockerWorkContainer, - infraContainerID: unknownInfraContainer, - netns: fmt.Sprintf("container:%s", unknownInfraContainer), - expectedErr: fmt.Errorf("allocated IP address not found"), - }, { - name: "docker-workload-no-endpoint", - podName: "pod4", - containerID: dockerWorkContainer, - infraContainerID: dockerInfraContainer, - netns: fmt.Sprintf("container:%s", dockerInfraContainer), - oriIPAMResult: oriIPAMResult, - errResponse: &cnipb.CniCmdResponse{ - Error: &cnipb.Error{ - Code: cnipb.ErrorCode_CONFIG_INTERFACE_FAILURE, - Message: "failed to find HNSEndpoint: pod4-6631b7", - }, - }, - }, { - name: "docker-workload-attach-failure", - podName: "pod5", - containerID: dockerWorkContainer, - infraContainerID: dockerInfraContainer, - netns: fmt.Sprintf("container:%s", dockerInfraContainer), - oriIPAMResult: oriIPAMResult, - endpointAttachErr: fmt.Errorf("unable to attach HnsEndpoint"), - endpointExists: true, - errResponse: &cnipb.CniCmdResponse{ - Error: &cnipb.Error{ - Code: cnipb.ErrorCode_CONFIG_INTERFACE_FAILURE, - Message: "failed to configure container IP: unable to attach HnsEndpoint", - }, - }, - }, { - name: "docker-workload-success", - podName: "pod6", - containerID: dockerWorkContainer, - infraContainerID: dockerInfraContainer, - netns: fmt.Sprintf("container:%s", dockerInfraContainer), - oriIPAMResult: oriIPAMResult, - endpointExists: true, - }, { - name: "docker-workload-already-attached", - podName: "pod7", - containerID: dockerWorkContainer, - infraContainerID: dockerInfraContainer, - netns: fmt.Sprintf("container:%s", dockerInfraContainer), - isAttached: true, - endpointExists: true, - oriIPAMResult: oriIPAMResult, - }, { name: "containerd-success", podName: "pod8", containerID: containerdInfraContainer, @@ -500,14 +381,9 @@ func TestCmdAdd(t *testing.T) { } ovsPortID := generateUUID() if tc.connectOVS { - if isDocker { - mockOVSBridgeClient.EXPECT().CreateInternalPort(ovsPortName, int32(0), gomock.Any(), gomock.Any()).Return(ovsPortID, nil).Times(1) - mockOVSBridgeClient.EXPECT().GetOFPort(ovsPortName, false).Return(int32(100), nil).Times(1) - } else { - mockOVSBridgeClient.EXPECT().CreatePort(ovsPortName, ovsPortName, gomock.Any()).Return(ovsPortID, nil).Times(1) - mockOVSBridgeClient.EXPECT().SetInterfaceType(ovsPortName, "internal").Return(nil).Times(1) - mockOVSBridgeClient.EXPECT().GetOFPort(ovsPortName, true).Return(int32(100), nil).Times(1) - } + mockOVSBridgeClient.EXPECT().CreatePort(ovsPortName, ovsPortName, gomock.Any()).Return(ovsPortID, nil).Times(1) + mockOVSBridgeClient.EXPECT().SetInterfaceType(ovsPortName, "internal").Return(nil).Times(1) + mockOVSBridgeClient.EXPECT().GetOFPort(ovsPortName, true).Return(int32(100), nil).Times(1) mockOFClient.EXPECT().InstallPodFlows(ovsPortName, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mockRoute.EXPECT().AddLocalAntreaFlexibleIPAMPodRule(gomock.Any()).Return(nil).Times(1) } @@ -577,22 +453,14 @@ func TestCmdDel(t *testing.T) { ifaceExists bool errResponse *cnipb.CniCmdResponse }{ - { - name: "docker-infra-success", - netns: "none", - ipamDel: true, - disconnectOVS: true, - endpointExists: true, - ifaceExists: true, - }, { name: "interface-not-exist", - netns: "none", + netns: generateUUID(), ipamDel: true, }, { name: "ipam-delete-failure", - netns: "none", + netns: generateUUID(), ipamDel: true, ipamError: fmt.Errorf("unable to delete IP"), disconnectOVS: true, @@ -667,6 +535,7 @@ func TestCmdCheck(t *testing.T) { ipam.ResetIPAMDriver(ipamType, ipamMock) ctx := context.TODO() + containerNetns := generateUUID() containerID := "261a1970-5b6c-11ed-8caf-000c294e5d03" mac, _ := net.ParseMAC("11:22:33:44:33:22") containerIP, containerIPNet, _ := net.ParseCIDR("10.1.2.100/24") @@ -709,11 +578,11 @@ func TestCmdCheck(t *testing.T) { { name: "check-success", podName: "pod0", - netns: "none", + netns: containerNetns, containerID: containerID, prevResult: wrapperIPAMResult(*ipamResult, []*current.Interface{ {Name: "pod0-6631b7", Mac: "11:22:33:44:33:22", Sandbox: ""}, - {Name: "pod0-6631b7_eth0", Mac: "11:22:33:44:33:22", Sandbox: "none"}, + {Name: "pod0-6631b7_eth0", Mac: "11:22:33:44:33:22", Sandbox: containerNetns}, }), existingIface: wrapperContainerInterface("pod0-6631b7", containerID, "pod0", generateUUID(), mac, containerIP), netInterface: &net.Interface{ @@ -725,7 +594,7 @@ func TestCmdCheck(t *testing.T) { }, { name: "pod-namespace-mismatch", podName: "pod1", - netns: "none", + netns: containerNetns, containerID: containerID, prevResult: wrapperIPAMResult(*ipamResult, []*current.Interface{ {Name: "pod1-6631b7", Mac: "11:22:33:44:33:22", Sandbox: ""}, @@ -741,17 +610,17 @@ func TestCmdCheck(t *testing.T) { errResponse: &cnipb.CniCmdResponse{ Error: &cnipb.Error{ Code: cnipb.ErrorCode_CHECK_INTERFACE_FAILURE, - Message: "sandbox in prevResult invalid-namespace doesn't match configured netns: none", + Message: fmt.Sprintf("sandbox in prevResult invalid-namespace doesn't match configured netns: %s", containerNetns), }, }, }, { name: "container-host-names-mismatch", podName: "pod2", - netns: "none", + netns: containerNetns, containerID: containerID, prevResult: wrapperIPAMResult(*ipamResult, []*current.Interface{ {Name: "pod2-6631b7", Mac: "11:22:33:44:33:22", Sandbox: ""}, - {Name: "eth0", Mac: "11:22:33:44:33:22", Sandbox: "none"}, + {Name: "eth0", Mac: "11:22:33:44:33:22", Sandbox: containerNetns}, }), existingIface: wrapperContainerInterface("pod2-6631b7", containerID, "pod2", generateUUID(), mac, containerIP), netInterface: &net.Interface{ @@ -769,11 +638,11 @@ func TestCmdCheck(t *testing.T) { }, { name: "container-host-MAC-mismatch", podName: "pod3", - netns: "none", + netns: containerNetns, containerID: containerID, prevResult: wrapperIPAMResult(*ipamResult, []*current.Interface{ {Name: "pod3-6631b7", Mac: "11:22:33:44:33:22", Sandbox: ""}, - {Name: "pod3-6631b7_eth0", Mac: "11:22:33:44:33:33", Sandbox: "none"}, + {Name: "pod3-6631b7_eth0", Mac: "11:22:33:44:33:33", Sandbox: containerNetns}, }), existingIface: wrapperContainerInterface("pod3-6631b7", containerID, "pod3", generateUUID(), mac, containerIP), netInterface: &net.Interface{