Update multi-net components to latest releases (#329)

* Update multi-net components to latest releases

Multus CNI 3.4.2
SR-IOV CNI 2.3
SR-IOV DP  3.2

Tested as per the README. Works fine.

Signed-off-by: Saikrishna Edupuganti <saikrishna.edupuganti@intel.com>

* Provide dpdk-stable 3 LTS and 1 recent release

Testpmd manifests of the last 3 LTS and 1 latest release from stable
repo to help test actual DPDK app instead of sleep.

Currently 17.11 and 19.11 are the only functioning ones without
privileged.

```
NAME              READY   STATUS    RESTARTS   AGE
dpdk-1711         1/1     Running   0          3m5s
dpdk-1811         0/1     Error     0          3m5s
dpdk-1911         1/1     Running   0          3m5s
dpdk-2002         0/1     Error     0          3m5s
```

```
EAL: PCI device 0000:07:06.4 on NUMA socket 0
EAL:   probe driver: 8086:154c net_i40e_vf
EAL: Getting a vfio_dev_fd for 0000:07:06.4 failed
EAL: Requested device 0000:07:06.4 cannot be used
…
testpmd: No probed ethernet devices
EAL: Error - exiting with code: 1
  Cause: Invalid port 1
```

Signed-off-by: Saikrishna Edupuganti <saikrishna.edupuganti@intel.com>

Author: krsna1729

clr-k8s-examples/9-multi-network/Dockerfile

@@ -1,20 +1,28 @@
 # Build multus plugin
-FROM golang:1.10 AS multus
-RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
-WORKDIR /go/src/github.com/intel/multus-cni
-RUN ./build
+FROM busybox AS multus
+ARG MULTUS_VER=3.4.2
+RUN wget -O multus.tgz https://github.com/intel/multus-cni/releases/download/v${MULTUS_VER}/multus-cni_${MULTUS_VER}_linux_amd64.tar.gz
+RUN tar xvzf multus.tgz --strip-components=1 -C /bin
 
 # Build sriov plugin
-FROM golang:1.10 AS sriov-cni
-RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
-WORKDIR /go/src/github.com/intel-corp/sriov-cni
-RUN ./build
+FROM golang AS sriov-cni
+ARG SRIOV_CNI_VER=2.3
+RUN wget -qO sriov-cni.tgz https://github.com/intel/sriov-cni/archive/v${SRIOV_CNI_VER}.tar.gz
+RUN mkdir -p sriov-cni && \
+    tar xzf sriov-cni.tgz --strip-components=1 -C sriov-cni && \
+    cd sriov-cni && \
+    make && \
+    cp build/sriov /bin
 
 # Build sriov device plugin
-FROM golang:1.10 AS sriov-dp
-RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
-WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
-RUN make
+FROM golang AS sriov-dp
+ARG SRIOV_DP_VER=3.2
+RUN wget -qO sriov-dp.tgz https://github.com/intel/sriov-network-device-plugin/archive/v${SRIOV_DP_VER}.tar.gz
+RUN mkdir -p sriov-dp && \
+    tar xzf sriov-dp.tgz --strip-components=1 -C sriov-dp && \
+    cd sriov-dp && \
+    make && \
+    cp build/sriovdp /bin
 
 # Build vfioveth plugin
 FROM busybox as vfioveth
@@ -25,9 +33,9 @@ RUN chmod +x /bin/vfioveth /bin/jq
 # Final image
 FROM centos/systemd
 WORKDIR /tmp/cni/bin
-COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
-COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
+COPY --from=multus /bin/multus-cni .
+COPY --from=sriov-cni /bin/sriov .
 COPY --from=vfioveth /bin/vfioveth .
 COPY --from=vfioveth /bin/jq .
 WORKDIR /usr/bin
-COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .
+COPY --from=sriov-dp /bin/sriovdp .

clr-k8s-examples/9-multi-network/README.md

@@ -9,11 +9,10 @@ directories on the host with the necessary binaries and configuration files.
 
 ### Customization
 
-The device plugin will register the SR-IOV enabled devices on the host, specified as
-`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
-file is provided, which enables SR-IOV for the above `rootDevices`
-
-> NOTE: This assumes homogenous nodes in the cluster
+The device plugin will register the SR-IOV enabled devices on the host, specified with
+`selectors` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
+file is provided, which enables SR-IOV for the above devices. More config options
+are listed [here](https://github.com/intel/sriov-network-device-plugin#configurations).
 
 ### Pre-req (SR-IOV only)
 

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -2,25 +2,34 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
   name: network-attachment-definitions.k8s.cni.cncf.io
 spec:
-  # group name to use for REST API: /apis/<group>/<version>
   group: k8s.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
   scope: Namespaced
   names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
     plural: network-attachment-definitions
-    # singular name to be used as an alias on the CLI and for display
     singular: network-attachment-definition
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
     kind: NetworkAttachmentDefinition
-    # shortNames allow shorter string to match your resource on the CLI
     shortNames:
     - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -40,17 +49,43 @@ type: kubernetes.io/service-account-token
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: multus-pod-networks-lister
+  name: multus-pod-updater
 rules:
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get"]
-- apiGroups: [""]
-  resources: ["pods/status"]
-  verbs: ["update"]
-- apiGroups: ["k8s.cni.cncf.io"]
-  resources: ["*"]
-  verbs: ["get"]
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-pod-updater
+subjects:
+- kind: ServiceAccount
+  name: multus-sa
+  namespace: kube-system
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -94,8 +129,9 @@ data:
     MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
     cat > /host/etc/cni/net.d/00-multus.conf <<EOF
     {
+      "cniVersion": "0.3.1",
       "name": "multus-cni-network",
-      "type": "multus",
+      "type": "multus-cni",
       "logFile": "/var/log/multus.log",
       "logLevel": "debug",
       "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
@@ -105,19 +141,6 @@ data:
     }
     EOF
 ---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus-rb
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus-pod-networks-lister
-subjects:
-- kind: ServiceAccount
-  name: multus-sa
-  namespace: kube-system
----
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -134,14 +157,14 @@ spec:
     spec:
       initContainers:
       - name: multus
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
         command: [ "bash", "-c" ]
         args:
-        - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
+        - cp /tmp/cni/bin/{multus-cni,sriov,vfioveth,jq} /host/opt/cni/bin/;
           /tmp/multus/install-multus-conf.sh;
           /tmp/multus/install-certs.sh;
           echo "Restarting crio kubelet";
-          systemctl restart crio; # Needed when crio manages ns lifecycle
+          systemctl restart crio;
           systemctl restart kubelet;
         volumeMounts:
         - name: usr-bin
@@ -160,7 +183,7 @@ spec:
           mountPath: /run/systemd
       containers:
       - name: sriovdp
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
         command: [ "sh", "-c" ]
         args:
         - /usr/bin/sriovdp --logtostderr -v 10;
@@ -209,4 +232,3 @@ spec:
           path: /var/lib/kubelet/device-plugins/
       hostNetwork: true
       hostPID: true
-

clr-k8s-examples/9-multi-network/sriov-conf.yaml

@@ -11,15 +11,15 @@ data:
       [
         {
           "resourceName": "sriov_netdevice",
-          "rootDevices": ["07:00.0"],
-          "sriovMode": true,
-          "deviceType": "netdevice"
+          "selectors": {
+                  "drivers": ["i40evf", "iavf"]
+          }
         },
         {
           "resourceName": "sriov_vfio",
-          "rootDevices": ["07:00.1"],
-          "sriovMode": true,
-          "deviceType": "vfio"
+          "selectors": {
+                  "drivers": ["vfio-pci"]
+          }
         }
       ]
     }

clr-k8s-examples/9-multi-network/test/sriov/2-pod-dpdk-ver.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1711
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:17.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1811
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:18.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1911
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:19.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-2002
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:20.02
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'

clr-k8s-examples/9-multi-network/test/sriov/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:bionic as ubuntu-build
+RUN apt-get update && \
+    apt-get -y install \
+        build-essential \
+        git \
+        libnuma-dev
+
+ARG DPDK_VER='master'
+ENV DPDK_DIR='/dpdk'
+ENV RTE_TARGET='x86_64-native-linuxapp-gcc'
+RUN git clone -b $DPDK_VER -q --depth 1 http://dpdk.org/git/dpdk-stable $DPDK_DIR 2>&1
+RUN cd ${DPDK_DIR} && \
+    sed -ri 's,(IGB_UIO=).*,\1n,' config/common_linux* && \
+    sed -ri 's,(KNI_KMOD=).*,\1n,' config/common_linux* && \
+    make config T=x86_64-native-linuxapp-gcc && \
+    make -j $CPUS
+ENV PATH="$PATH:$DPDK_DIR/build/app/"