Skip to content

Commit 5a5f46d

Browse files
authored
Allow more devices per gateway setup (#917)
1 parent 556b137 commit 5a5f46d

30 files changed

Lines changed: 464 additions & 186 deletions

.github/workflows/tests.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,10 @@ jobs:
6363
name: build_artifacts_base
6464
path: /tmp/build_artifacts_base
6565
- uses: actions/download-artifact@v2
66+
if: matrix.test == 'aux'
6667
with:
6768
name: build_artifacts_subset
6869
path: /tmp/build_artifacts_subset
69-
if: matrix.test == 'aux'
7070
- uses: actions/checkout@v2
7171
with:
7272
fetch-depth: 0
@@ -87,9 +87,9 @@ jobs:
8787
run: |
8888
bin/load_images faucet faux setup base
8989
- name: Loading subset docker images
90+
if: matrix.test == 'aux'
9091
run: |
9192
bin/load_images subset
92-
if: matrix.test == 'aux'
9393
- name: Running ${{ matrix.test }} test
9494
env:
9595
DOCKER_STARTUP_TIMEOUT_MS: 60000
@@ -176,7 +176,7 @@ jobs:
176176
bin/setup_remote daq
177177
bin/retry_cmd bin/build_docker controller
178178
bin/build_dts
179-
- name: run fot integration tests
179+
- name: run vxlan integration tests
180180
if: ${{ always() }}
181181
run: |
182182
cd forch

bin/catwrap

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
#
3+
# Same problem as cited here:
4+
# https://discuss.linuxcontainers.org/t/tcpdump-inside-a-container-failed/6637
5+
#
6+
# Really not sure what's going on, but piping the output through cat fixes the problem.
7+
#
8+
9+
$@ 2>&1 | cat

bin/setup_dev

Lines changed: 9 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ if [[ $PVERSION != $INSTALLED_PYTHON ]]; then
2727
false
2828
fi
2929

30-
MININET=https://github.com/mininet/mininet
31-
MININETV=$(cat etc/MININET_VERSION)
32-
3330
if [ -f .daq.local ]; then
3431
echo Loading config from .daq.local
3532
source .daq.local
@@ -73,39 +70,7 @@ $AG install \
7370
isc-dhcp-client network-manager netcat gnupg2 strace arp-scan libffi-dev \
7471
python python$PVERSION python3-pkg-resources python3-setuptools \
7572
python$PVERSION-dev python3-pip python$PVERSION-venv \
76-
python3-distutils
77-
78-
PYTHON2_BIN=$(readlink -f $(which python2))
79-
echo Python2 binary is $PYTHON2_BIN
80-
81-
if [ -d mininet ]; then
82-
echo Checking mininet version matches $MININETV...
83-
targetrev=$(cd mininet; git rev-parse $MININETV)
84-
instrev=$(cd mininet; git rev-parse HEAD)
85-
if [ "$targetrev" != "$instrev" ]; then
86-
echo Target mininet version $MININETV does not match installed version.
87-
false
88-
fi
89-
if [ ! -f mininet/.the_house_that_daq_built ]; then
90-
echo Mininet build seems to be incomplete. Try bin/clean_dev and try again.
91-
false
92-
fi
93-
else
94-
echo Cloning $MININET $MININETV...
95-
git clone $MININET
96-
(
97-
cd mininet
98-
git reset --hard $MININETV
99-
for i in ssh pep8 pyflakes python-pexpect pylint xterm ; do
100-
perl -pi -e "s/${i}//g" util/install.sh ;
101-
done
102-
sed -i s/cgroup-bin/cgroup-tools/ util/install.sh
103-
export PYTHON2_BIN
104-
sed -i 's/sudo PYTHON=${PYTHON} make install/sudo PYTHON=${PYTHON2_BIN} make install/' util/install.sh
105-
util/install.sh -n
106-
)
107-
touch mininet/.the_house_that_daq_built
108-
fi
73+
python3-distutils golang-go protobuf-compiler
10974

11075
# Can't use venv inside of containers because of absolute paths.
11176
if [ -n "$CI" ]; then
@@ -142,30 +107,31 @@ $PIP freeze
142107
echo Resetting .cache directory permissions...
143108
test -n "$USER" && sudo chown $USER -R $HOME/.cache
144109

110+
bin/setup_mininet
111+
145112
if [ -n "$SIMPLE" ]; then
146113
echo Finished with simple setup.
147114
exit 0
148115
fi
149116

117+
rm -rf protoc-gen-doc
118+
git clone https://github.com/pseudomuto/protoc-gen-doc.git
119+
150120
bin/setup_remote faucet
151121
bin/setup_remote forch
152122
bin/setup_remote udmi
153123

154-
echo -n "DAQ commit "
155-
git log -n 1 --pretty=format:"%h - %an, %ar : %s" || true
156-
echo
157-
158124
docker --version
159-
160125
if ! docker images > /dev/null; then
161126
echo
162127
echo Docker execution failed, is the docker group setup?
163128
echo If this is the first time, try logging out and log back in again.
164129
false
165130
fi
166131

167-
rm -rf protoc-gen-doc
168-
git clone https://github.com/pseudomuto/protoc-gen-doc.git
132+
echo -n "DAQ commit "
133+
git log -n 1 --pretty=format:"%h - %an, %ar : %s" || true
134+
echo
169135

170136
echo
171137
echo Setup completed successfully.

bin/setup_mininet

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash -e
2+
3+
ROOT=$(dirname $0)/..
4+
cd $ROOT
5+
6+
MININET=https://github.com/mininet/mininet
7+
MININETV=$(cat etc/MININET_VERSION)
8+
9+
source venv/bin/activate
10+
11+
PYTHON2_BIN=$(readlink -f $(which python2))
12+
echo Python2 binary is $PYTHON2_BIN
13+
14+
#
15+
# If there's trouble installing this because of setuptools, then some
16+
# manual trickery needs to be done to install setuptools for python2.
17+
# This just involves downloading it manually for python2, and then
18+
# running pip for python2 with setuptools. This link should help:
19+
#
20+
# https://stackoverflow.com/a/66719099
21+
#
22+
# And then ultimately run the command:
23+
#
24+
# python -m pip install setuptools
25+
#
26+
# This isn't automated because it's only required on some legacy systems,
27+
# and might involve a security hole (downloading packages directly).
28+
#
29+
30+
if [ -d mininet ]; then
31+
echo Checking mininet version matches $MININETV...
32+
targetrev=$(cd mininet; git rev-parse $MININETV)
33+
instrev=$(cd mininet; git rev-parse HEAD)
34+
if [ "$targetrev" != "$instrev" ]; then
35+
echo Target mininet version $MININETV does not match installed version.
36+
false
37+
fi
38+
if [ ! -f mininet/.the_house_that_daq_built ]; then
39+
echo Mininet build seems to be incomplete. Try bin/clean_dev and try again.
40+
false
41+
fi
42+
else
43+
echo Cloning $MININET $MININETV...
44+
git clone $MININET
45+
(
46+
cd mininet
47+
git reset --hard $MININETV
48+
for i in ssh pep8 pyflakes python-pexpect pylint xterm ; do
49+
perl -pi -e "s/${i}//g" util/install.sh ;
50+
done
51+
sed -i s/cgroup-bin/cgroup-tools/ util/install.sh
52+
export PYTHON2_BIN
53+
sed -i 's/sudo PYTHON=${PYTHON} make install/sudo PYTHON=${PYTHON2_BIN} make install/' util/install.sh
54+
util/install.sh -n
55+
)
56+
touch mininet/.the_house_that_daq_built
57+
fi
58+

config/modules/ata.conf

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Testing include files for ATA runs.
2+
3+
# Tests to build.
4+
build ${DAQ_LIB}/docker/modules
5+
6+
add pass first
7+
add fail first
8+
add ping first
9+
add nmap

config/modules/dts.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Baseline test file for officially included tests.
1+
# Baseline test file for DTS hosted tests.
22

33
# Tests to build.
44
build ${DAQ_LIB}/docker/modules

config/system/ata.yaml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ include: ${DAQ_LIB}/config/system/default.yaml
66
# Description for dashboard.
77
site_description: "ATA Proxy Configuration"
88

9+
# Configure with proper set of tests.
10+
host_tests: config/modules/ata.conf
11+
912
# For large-scale network, no room for sleeping.
1013
settle_sec: 0
1114

@@ -19,7 +22,10 @@ switch_setup:
1922

2023
run_trigger:
2124
native_vlan: 122
22-
max_hosts: 3
25+
max_hosts: 10
26+
device_block_sec: 3600
27+
retain_results: True
28+
arp_scan_sec: 600
2329

2430
internal_subnet:
2531
subnet: 10.21.0.0/16

daq/base_gateway.py

Lines changed: 48 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,14 @@
1616

1717
LOGGER = logger.get_logger('gateway')
1818

19+
DEFAULT_TEST_HOSTS = 8
1920

2021
class BaseGateway(ABC):
2122
"""Gateway collection class for managing testing services"""
2223

2324
GATEWAY_OFFSET = 0
2425
FAKE_HOST_OFFSET = 1
2526
TEST_OFFSET_START = 2
26-
NUM_SET_PORTS = 6
27-
SET_SPACING = 10
2827
_PING_RETRY_COUNT = 5
2928

3029
TEST_IP_FORMAT = '192.168.84.%d'
@@ -45,6 +44,10 @@ def __init__(self, runner, name, port_set, env_params=None):
4544
self.activated = False
4645
self.result_linger = False
4746
self.dhcp_monitor = None
47+
is_native = bool(self.runner.run_trigger.get('native_vlan'))
48+
max_hosts = self.runner.run_trigger.get('max_hosts') or DEFAULT_TEST_HOSTS
49+
num_host_ports = max_hosts if is_native else DEFAULT_TEST_HOSTS
50+
self._gw_set_size = num_host_ports + self.TEST_OFFSET_START
4851
self._env_params = env_params
4952
self._ext_intf = env_params.get('ext_intf') if env_params else None
5053
self._is_native = bool(self._ext_intf)
@@ -136,7 +139,7 @@ def allocate_test_port(self):
136139
test_port = self._switch_port(self.TEST_OFFSET_START)
137140
while test_port in self.test_ports:
138141
test_port = test_port + 1
139-
limit_port = self._switch_port(self.NUM_SET_PORTS)
142+
limit_port = self._switch_port(self._gw_set_size)
140143
assert test_port < limit_port, 'no test ports available'
141144
self.test_ports.add(test_port)
142145
return test_port
@@ -147,15 +150,10 @@ def release_test_port(self, test_port):
147150
self.test_ports.remove(test_port)
148151

149152
def _switch_port(self, offset):
150-
return self.port_set * self.SET_SPACING + offset
153+
return self.port_set * self._gw_set_size + offset
151154

152155
def _is_target_expected(self, target):
153-
if not target:
154-
return False
155-
target_mac = target['mac']
156-
if target_mac in self.targets:
157-
return True
158-
return False
156+
return target and target['mac'] in self.targets
159157

160158
def _dhcp_callback(self, state, target, exception=None):
161159
if exception:
@@ -175,16 +173,16 @@ def _setup_tmpdir(self, base_name):
175173
def attach_target(self, device):
176174
"""Attach the given target to this gateway; return number of attached targets."""
177175
assert device.mac not in self.targets, 'target %s already attached to gw' % device
178-
LOGGER.info('Attaching target %s to gateway group %s',
179-
device, self.name)
176+
LOGGER.info('Attaching target %s to gateway %s group %s',
177+
device, self.port_set, self.name)
180178
self.targets[device.mac] = device
181179
return len(self.targets)
182180

183181
def detach_target(self, device):
184182
"""Detach the given target from this gateway; return number of remaining targets."""
185183
assert device.mac in self.targets, 'target %s not attached to gw' % device
186-
LOGGER.info('Detach target %s from gateway group %s: %s',
187-
device, self.name, list(self.targets.keys()))
184+
LOGGER.info('Detaching target %s from gateway %s group %s: %s',
185+
device, self.port_set, self.name, list(self.targets.keys()))
188186
del self.targets[device.mac]
189187
return len(self.targets)
190188

@@ -200,10 +198,16 @@ def get_targets(self):
200198
"""Return the host targets associated with this gateway"""
201199
return self.targets.values()
202200

201+
def get_all_gw_ports(self):
202+
"""Return all ports associated with gateway"""
203+
base_port = self._switch_port(0)
204+
limit_port = self._switch_port(self._gw_set_size)
205+
return list(range(base_port, limit_port))
206+
203207
def get_possible_test_ports(self):
204208
"""Return test ports associated with gateway"""
205209
test_port = self._switch_port(self.TEST_OFFSET_START)
206-
limit_port = self._switch_port(self.NUM_SET_PORTS)
210+
limit_port = self._switch_port(self._gw_set_size)
207211
return list(range(test_port, limit_port))
208212

209213
def terminate(self):
@@ -233,7 +237,7 @@ def terminate(self):
233237
def _get_scan_interface(self):
234238
return self.host, self.host_intf
235239

236-
def _discover_host_hangup_callback(self, mac, log_fd, log_file, callback):
240+
def _discover_host_hangup_callback(self, mac, log_fd, log_file, scan_callback):
237241
def process_line(line):
238242
sections = [section for section in line.split('\t') if section]
239243
if len(sections) >= 2:
@@ -252,38 +256,55 @@ def process_line(line):
252256
if device_ip:
253257
LOGGER.info('Host discovery for %s completed. Found ip %s.',
254258
mac, device_ip)
255-
return callback(device_ip)
259+
return scan_callback(device_ip)
256260
LOGGER.info('Host discovery for %s completed. Found no ip.', mac)
257-
callback(None)
261+
scan_callback(None)
258262

259-
def discover_host(self, mac: str, subnets: List[ip_network], callback: Callable):
263+
def discover_host(self, mac: str, all_subnets: List[ip_network], host_callback: Callable):
260264
"""Discovers a host using arp-scan in a list of subnets."""
261265
cmd = 'arp-scan --retry=2 --bandwidth=512K --interface=%s --destaddr=%s -s %s %s'
262266
host, intf = self._get_scan_interface()
267+
scans = self.runner.run_trigger.get('arp_scan_count') or 2
268+
if scans <= 0:
269+
return
270+
subnets = list(all_subnets)
263271
LOGGER.info('Starting host discovery for %s', mac)
264272

265-
def hangup_callback_callback(device_ip):
273+
def scan_callback(device_ip):
274+
nonlocal scans
266275
if device_ip:
267-
callback(device_ip)
276+
scans = 0
277+
host_callback(device_ip)
268278
else:
269279
recursive_discover()
270280

271281
def recursive_discover():
282+
nonlocal subnets, scans
272283
if not subnets:
273-
callback(None)
274-
return
284+
if scans > 0:
285+
scans -= 1
286+
subnets = list(all_subnets)
287+
else:
288+
host_callback(None)
289+
return
275290
subnet = subnets.pop(0)
276-
address = next(subnet.hosts())
291+
hosts = subnet.hosts()
292+
address = next(hosts)
293+
# This handles the case where the local IP conflicts with the remote.
294+
# Not really a good general solution, but works in many cases.
295+
# TODO: Generalize a bit more to (e.g.) pick a random src IP.
296+
if scans % 2:
297+
address = next(hosts)
277298
log_file = os.path.join(self.tmpdir, str(subnet).replace('/', '_'))
278299
log_fd = open(log_file, 'w')
279-
LOGGER.info('Scanning subnet %s from %s for %s', subnet, address, mac)
300+
LOGGER.info('Scanning %s subnet %s from %s for %s', scans, subnet, address, mac)
280301
host.cmd('ip addr add %s/%s dev %s' % (str(address), subnet.prefixlen, intf))
281302
full_cmd = cmd % (intf, mac, str(address), str(subnet))
282-
LOGGER.info('arp-scan command: %s', full_cmd)
283303
active_pipe = host.popen(full_cmd, stdin=DEVNULL, stdout=PIPE, env=os.environ)
284304
self.runner.monitor_stream(self.name, active_pipe.stdout, copy_to=log_fd,
285305
hangup=lambda: self._discover_host_hangup_callback(
286-
mac, log_fd, log_file, hangup_callback_callback))
306+
mac, log_fd, log_file, scan_callback))
307+
287308
recursive_discover()
288309

289310
def _ping_test(self, src, dst, src_addr=None):

0 commit comments

Comments
 (0)