diff --git a/docker-compose/b2b-pinning/Makefile b/docker-compose/b2b-pinning/Makefile new file mode 100644 index 00000000..538b91fe --- /dev/null +++ b/docker-compose/b2b-pinning/Makefile @@ -0,0 +1,101 @@ +SHELL = /bin/bash + +.PHONY: all +all: install network deploy run + +.PHONY: all-nopin +all-nopin: install network deploy-nopin run + +.PHONY: all-argpin +all-argpin: install network deploy-argpin run + +.PHONY: clean +clean: remove-lab network-clean + +.PHONY: clean-nopin +clean-nopin: remove-lab-nopin network-clean + +.PHONY: clean-argpin +clean-argpin: remove-lab-argpin network-clean + +.PHONY: clean-all +clean-all: clean clean-nopin clean-argpin install-clean + +############################### +# Install components +############################### + +.PHONY: install +install: install-docker-compose install-otgen + +install-docker-compose: /usr/local/bin/docker-compose +/usr/local/bin/docker-compose: + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$$(uname -s)-$$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + +install-otgen: /usr/local/bin/otgen +/usr/local/bin/otgen: + bash -c "$$(curl -sL https://get.otgcdn.net/otgen)" -- -v 0.4.0 + +install-clean: + -sudo rm -f `command -v docker-compose` + -sudo rm -f `command -v otgen` + +############################### +# Test network +############################### + +.PHONY: network +network:veth0 + +veth0: /sys/class/net/veth0 +/sys/class/net/veth0: + sudo ip link add name veth0 type veth peer name veth1 + sudo ip link set dev veth0 up + sudo ip link set dev veth1 up + +network-clean: veth0-clean +veth0-clean: + -sudo ip link del name veth0 type veth peer name veth1 + +############################### +# Deploy lab +############################### + +.PHONY: deploy +deploy: deploy-lab + +.PHONY: deploy-nopin +deploy-nopin: deploy-lab-nopin + +.PHONY: deploy-argpin +deploy-argpin: deploy-lab-argpin + +deploy-lab: + sudo docker-compose up -d + +remove-lab: + -sudo docker-compose down + +deploy-lab-nopin: + sudo docker-compose -f compose.nopin.yml up -d + +remove-lab-nopin: + -sudo docker-compose -f compose.nopin.yml down + +deploy-lab-argpin: + sudo docker-compose -f compose.argpin.yml up -d + +remove-lab-argpin: + -sudo docker-compose -f compose.argpin.yml down + + +############################### +# Run tests +############################### + +.PHONY: run +run: otgen-run + +otgen-run: + otgen run -f otg.yml -k --api https://localhost:8443 -m flow | otgen transform -m flow -c pps | otgen display -m table diff --git a/docker-compose/b2b-pinning/README.md b/docker-compose/b2b-pinning/README.md new file mode 100644 index 00000000..d0230b48 --- /dev/null +++ b/docker-compose/b2b-pinning/README.md @@ -0,0 +1,179 @@ +# Ixia-c traffic engine back-to-back setup with CPU Core Pinning + +## Overview + +This lab demonstrates how to maximize performance of [Ixia-c](https://github.com/open-traffic-generator/ixia-c) Traffic Engine ports via CPU pinning. The lab has two traffic ports connected back-2-back using a veth pair. The lab is defined via Docker Compose YAML file. + +![Diagram](./diagram.png) + +## Deployment Details + +In the deployment steps listed below, two `traffic-engine` containers will be started. The table below describes parameters that are unique among them: + +- **Listen Port**: Port on which the `traffic-engine` will listen to for communication with the `controller` +- **Network Interfaces**: Interfaces on which the `traffic-engine` will transmit and receive traffic +- **CPU Cores Allocated**: Each instance of `traffic-engine` uses three cores. Two dedicated cores are used for + sending and receiving packets and one is used for configuration, control and statistics. In the example above all + instances of `traffic-engine` are sharing core number 2 for the configuration, control and statistics while + instance 1 is using cores 3/4 for Tx/Rx, instance 2 is using cores 5/6 for Tx/Rx, and so on + +| Instance | Listen Port | Cores Allocated | Network Interfaces | +|----------|-------------|-----------------|-------------------------| +| 1 | 5555 | 2, 3, 4 | veth0 | +| 2 | 5556 | 2, 5, 6 | veth1 | + + +**Note**: The Tx/Rx cores allocated to a `traffic-engine` instance using `cpuset` parameter MUST NOT overlap across other such instances (even if they are idle and not sending/receiving any traffic). Sharing Tx/Rx cores will result in undefined behavior including but not limited to loss of performance. Further due to a NUMA-related limitation with DPDK 19.11 used by Ixia-C Traffic Engine, the core used for control needs to be same across all `traffic-engine` instances. In the example above, core number 2 is used for control in all three containers. + + + ```Yaml + services: + controller: + image: ghcr.io/open-traffic-generator/ixia-c-controller:0.0.1-3724 + command: --accept-eula --http-port 8443 + network_mode: "host" + restart: always + traffic_engine_1: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + cpuset: 2,3,4 + environment: + - OPT_LISTEN_PORT=5555 + - ARG_IFACE_LIST=virtual@af_packet,veth0 + - OPT_NO_HUGEPAGES=Yes + traffic_engine_2: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + cpuset: 2,5,6 + environment: + - OPT_LISTEN_PORT=5556 + - ARG_IFACE_LIST=virtual@af_packet,veth1 + - OPT_NO_HUGEPAGES=Yes + ``` + +## System Prerequisites + +### CPU and RAM + +* `traffic-engine` - each instance requires 2 dedicated CPU cores and 3GB dedicated RAM. +* `controller` - each instance requires at least 1 CPU core and 2GB RAM. + +### Platform + +* x86_64 Linux Distribution (Centos 7+ or Ubuntu 18+) with sudo permissions and Docker support +* [Docker 19+](https://docs.docker.com/engine/install/) + +## Install components + +1. Install `docker-compose` + + ```Shell + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + ``` + +2. Install `otgen` + + ```Shell + curl -L "https://github.com/open-traffic-generator/otgen/releases/download/v0.2.0/otgen_0.2.0_$(uname -s)_$(uname -m).tar.gz" | tar xzv otgen + sudo mv otgen /usr/local/bin/otgen + sudo chmod +x /usr/local/bin/otgen + ``` + +3. Make sure `/usr/local/bin` is in your `$PATH` variable (by default this is not the case on CentOS 7) + + ```Shell + cmd=docker-compose + dir=/usr/local/bin + if ! command -v ${cmd} &> /dev/null && [ -x ${dir}/${cmd} ]; then + echo "${cmd} exists in ${dir} but not in the PATH, updating PATH to:" + PATH="/usr/local/bin:${PATH}" + echo $PATH + fi + ``` + +4. Clone this repository (optional, only needed to use `make all` to run all the steps automatically) + + ```Shell + git clone --recursive https://github.com/open-traffic-generator/otg-examples.git + cd otg-examples/docker-compose/b2b-pinning + ``` + +## Deploy Ixia-c lab + +1. Create veth pair `veth0 - veth1` + + ```Shell + sudo ip link add name veth0 type veth peer name veth1 + sudo ip link set dev veth0 up + sudo ip link set dev veth1 up + ``` + +2. Launch the deployment + + ```Shell + sudo docker-compose up -d + ``` + +4. Make sure you have all three containers running, check CPU pinning + + ```Shell + sudo docker ps + docker logs cpupin_traffic_engine_1_1 2>&1 | grep cores + docker logs cpupin_traffic_engine_2_1 2>&1 | grep cores + ``` + +## Run OTG traffic flows + +1. Download an example of OTG traffic flow configuration file: + + ```Shell + wget https://raw.githubusercontent.com/open-traffic-generator/otg-examples/main/docker-compose/b2b/otg.yml + ``` + +2. Start with using `otgen` to request Ixia-c to run traffic flows defined in `otg.yml`. If successful, the result will come as OTG port metrics in JSON format + + ```Shell + cat otg.yml | otgen run -k -a https://localhost:8443 + ``` + +3. You can now repeat this exercise, but transform output to a table + + ```Shell + cat otg.yml | otgen run -k -a https://localhost:8443 | otgen transform -m port | otgen display -m table + ``` + +4. The same, but with flow metrics + + ```Shell + cat otg.yml | otgen run -k -a https://localhost:8443 -m flow | otgen transform -m flow | otgen display -m table + ``` + +5. The same, but with byte instead of frame count (only receive stats are reported) + + ```Shell + cat otg.yml | otgen run -k -a https://localhost:8443 -m flow | otgen transform -m flow -c bytes | otgen display -m table + ``` + +6. Now report packet per second rate, as a line chart (end with `Crtl-c`) + + ```Shell + cat otg.yml | otgen run -k -a https://localhost:8443 -m flow | otgen transform -m flow -c pps | otgen display -m chart + ``` + +## Destroy the lab + +To destroy the lab, including veth pair, use: + +```Shell +docker-compose down +sudo ip link del name veth0 type veth peer name veth1 +``` + +## Credits + +* [Diana Galan](https://github.com/dgalan-xxia) is an author of `compose.yml` example. diff --git a/docker-compose/b2b-pinning/compose.argpin.yml b/docker-compose/b2b-pinning/compose.argpin.yml new file mode 100644 index 00000000..c36076c2 --- /dev/null +++ b/docker-compose/b2b-pinning/compose.argpin.yml @@ -0,0 +1,28 @@ +services: + controller: + image: ghcr.io/open-traffic-generator/ixia-c-controller:0.0.1-3724 + command: --accept-eula --http-port 8443 + network_mode: "host" + restart: always + traffic_engine_1: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + cpuset: 2,3,4 + environment: + - OPT_LISTEN_PORT=5555 + - ARG_IFACE_LIST=virtual@af_packet,veth0 + - ARG_CORE_LIST=2 3 4 + - OPT_NO_HUGEPAGES=Yes + traffic_engine_2: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + cpuset: 2,5,6 + environment: + - OPT_LISTEN_PORT=5556 + - ARG_IFACE_LIST=virtual@af_packet,veth1 + - ARG_CORE_LIST=2 5 6 + - OPT_NO_HUGEPAGES=Yes diff --git a/docker-compose/b2b-pinning/compose.nopin.yml b/docker-compose/b2b-pinning/compose.nopin.yml new file mode 100644 index 00000000..fdefff16 --- /dev/null +++ b/docker-compose/b2b-pinning/compose.nopin.yml @@ -0,0 +1,26 @@ +services: + controller: + image: ghcr.io/open-traffic-generator/ixia-c-controller:0.0.1-3724 + command: --accept-eula --http-port 8443 + network_mode: "host" + restart: always + traffic_engine_1: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + environment: + - OPT_LISTEN_PORT=5555 + - ARG_IFACE_LIST=virtual@af_packet,veth0 + - OPT_NO_HUGEPAGES=Yes + - OPT_NO_PINNING=Yes + traffic_engine_2: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + environment: + - OPT_LISTEN_PORT=5556 + - ARG_IFACE_LIST=virtual@af_packet,veth1 + - OPT_NO_HUGEPAGES=Yes + - OPT_NO_PINNING=Yes diff --git a/docker-compose/b2b-pinning/compose.yml b/docker-compose/b2b-pinning/compose.yml new file mode 100644 index 00000000..20806462 --- /dev/null +++ b/docker-compose/b2b-pinning/compose.yml @@ -0,0 +1,26 @@ +services: + controller: + image: ghcr.io/open-traffic-generator/ixia-c-controller:0.0.1-3724 + command: --accept-eula --http-port 8443 + network_mode: "host" + restart: always + traffic_engine_1: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + cpuset: 2,3,4 + environment: + - OPT_LISTEN_PORT=5555 + - ARG_IFACE_LIST=virtual@af_packet,veth0 + - OPT_NO_HUGEPAGES=Yes + traffic_engine_2: + image: ghcr.io/open-traffic-generator/ixia-c-traffic-engine:1.6.0.24 + network_mode: "host" + restart: always + privileged: true + cpuset: 2,5,6 + environment: + - OPT_LISTEN_PORT=5556 + - ARG_IFACE_LIST=virtual@af_packet,veth1 + - OPT_NO_HUGEPAGES=Yes diff --git a/docker-compose/b2b-pinning/diagram.png b/docker-compose/b2b-pinning/diagram.png new file mode 100644 index 00000000..2c58aa5b Binary files /dev/null and b/docker-compose/b2b-pinning/diagram.png differ diff --git a/docker-compose/b2b-pinning/otg.yml b/docker-compose/b2b-pinning/otg.yml new file mode 100644 index 00000000..4505d3d6 --- /dev/null +++ b/docker-compose/b2b-pinning/otg.yml @@ -0,0 +1,58 @@ +flows: +- name: p1-p2 + duration: + choice: fixed_packets + fixed_packets: + gap: 12 + packets: 1000000 + rate: + choice: pps + pps: 1000000 + size: + choice: fixed + fixed: 128 + metrics: + enable: true + loss: false + timestamps: false + packet: + - choice: ethernet + ethernet: + dst: + choice: value + value: 00:00:00:00:00:bb + src: + choice: value + value: 00:00:00:00:00:aa + - choice: ipv4 + ipv4: + dst: + choice: value + value: 192.0.2.1 + src: + choice: increment + increment: + start: 1.1.1.1 + step: 0.1.2.3 + count: 100 + - choice: tcp + tcp: + dst_port: + choice: value + value: 80 + src_port: + choice: increment + increment: + start: 23250 + step: 7 + count: 100 + tx_rx: + choice: port + port: + tx_name: p1 + rx_name: p2 +ports: +- location: localhost:5555 + name: p1 +- location: localhost:5556 + name: p2 \ No newline at end of file