Merge pull request #500 from NetSys/2to3

Python 2/3 Compatibility

Author: sangjinhan

.travis.yml

@@ -31,8 +31,9 @@ before_install:
   - sudo apt-get -q update
 
 install:
-  - sudo apt-get install -y python
-  - pip install grpcio scapy codecov
+  - sudo apt-get install -y python2.7 python3 python3-pip
+  - pip2 install grpcio scapy codecov
+  - pip3 install grpcio scapy-python3 coverage
   - "[[ ${DEBUG:-0} == 0 ]] || sudo apt-get install -y g++-5"  # install gcov-5
   - "[[ ${SANITIZE:-0} == 0 ]] || sudo apt-get install -y llvm-3.8"
   - "[[ $TAG_SUFFIX != _32 ]] || sudo apt-get install -y lib32gcc1"
@@ -50,8 +51,10 @@ script:
   - ./container_build.py bess
   - ./container_build.py kmod_buildtest
   - (cd core && ./all_test)  # TcpFlowReconstructTest requires working directory to be `core/`
-  - coverage run -m unittest discover -v
-  - bessctl/bessctl -- daemon start -- run testing/run_module_tests
+  - coverage2 run -m unittest discover -v
+  - "[[ ${DEBUG:-0} == 0 ]] || coverage3 run -m unittest discover -v"
+  - python2 bessctl/bessctl -- daemon start -- run testing/run_module_tests
+  - "[[ ${DEBUG:-0} == 0 ]] || python3 bessctl/bessctl -- daemon start -- run testing/run_module_tests"
 
 after_success:
   - bessctl/bessctl daemon stop # flush out the coverage data

bessctl/bessctl

@@ -1,24 +1,25 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
+from __future__ import print_function
+from __future__ import absolute_import
 import sys
 import os
 import os.path
-import cStringIO
+import io
 import tempfile
 import time
+import cli
+import commands
 
 # Suppress scapy IPv6 warning (must be done before importing scapy module)
 import logging
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 
-import cli
-import commands
-
 try:
     this_dir = os.path.dirname(os.path.realpath(__file__))
     sys.path.insert(1, os.path.join(this_dir, '..'))
     from pybess.bess import *
 except ImportError:
-    print >> sys.stderr, 'Cannot import the API module (pybess)'
+    print('Cannot import the API module (pybess)', file=sys.stderr)
     raise
 
 
@@ -68,7 +69,7 @@ class BESSCLI(cli.CLI):
 
         except self.bess.RPCError as e:
             self.err('RPC failed to {}:{} - {}'.format(
-                self.bess.peer[0], self.bess.peer[1], e.message))
+                self.bess.peer[0], self.bess.peer[1], str(e)))
 
             self._handle_broken_connection()
             raise self.HandledError()
@@ -120,7 +121,7 @@ def run_cli(instream=sys.stdin):
         s.connect()
     except BESS.APIError as e:
         if cli.interactive:
-            cli.ferr.write(e.message + '\n')
+            cli.ferr.write(str(e) + '\n')
             cli.ferr.write('Perhaps bessd daemon is not running locally? '
                            'Try "daemon start".\n')
 
@@ -147,8 +148,8 @@ def main():
             else:
                 line_buf.append(arg)
 
-        cmds.append(' '.join(line_buf))
-        run_cli(cStringIO.StringIO('\n'.join(cmds)))
+        cmds.append(u' '.join(line_buf))
+        run_cli(io.StringIO('\n'.join(cmds)))
 
 if __name__ == '__main__':
     main()

bessctl/cli.py

@@ -198,7 +198,7 @@ def list_matched(self, line, filters):
         if len(matched_list) == 0:
             return [], []
 
-        max_score = max(map(lambda x: x[1], matched_list))
+        max_score = max([x[1] for x in matched_list])
 
         ret = []
         ret_low = []
@@ -371,7 +371,11 @@ def print_banner(self):
     def process_one_line(self):
         if self.interactive:
             try:
-                line = raw_input(self.get_prompt())
+                # Hack for Python 2/3 compatibility
+                if hasattr(__builtins__, 'raw_input'):
+                    line = raw_input(self.get_prompt())
+                else:
+                    line = input(self.get_prompt())
             except KeyboardInterrupt:
                 self.fout.write('\n')
                 return

bessctl/commands.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+from __future__ import absolute_import
 import os
 import os.path
 import sys
@@ -15,7 +17,6 @@
 import tempfile
 import signal
 import collections
-
 import sugar
 
 try:
@@ -24,7 +25,7 @@
     from pybess.module import *
     from pybess.port import *
 except ImportError:
-    print >> sys.stderr, 'Cannot import the API module (pybess)'
+    print('Cannot import the API module (pybess)', file=sys.stderr)
     raise
 
 
@@ -62,8 +63,8 @@ def __bess_env__(key, default=None):
         if default is None:
             raise ConfError('Environment variable "%s" must be set.')
 
-        print 'Environment variable "%s" is not set. \
-            Using default value "%s"' % (key, default)
+        print('Environment variable "%s" is not set. \
+              Using default value "%s"' % (key, default))
         return default
 
 
@@ -545,7 +546,12 @@ def warn(cli, msg, func, *args):
             cli.rl.set_completer(cli.complete_dummy)
 
         try:
-            resp = raw_input('WARNING: %s Are you sure? (type "yes") ' % msg)
+            # Hack for Python 2/3 compatibility
+            if hasattr(__builtins__, 'raw_input'):
+                resp = raw_input(
+                    'WARNING: %s Are you sure? (type "yes") ' % msg)
+            else:
+                resp = input('WARNING: %s Are you sure? (type "yes") ' % msg)
 
             if resp.strip() == 'yes':
                 func(cli, *args)
@@ -742,13 +748,13 @@ def _run_file(cli, conf_file, env_map):
         try:
             original_env = copy.copy(os.environ)
 
-            for k, v in env_map.iteritems():
+            for k, v in env_map.items():
                 os.environ[k] = str(v)
 
             _do_run_file(cli, conf_file)
         finally:
             os.environ.clear()
-            for k, v in original_env.iteritems():
+            for k, v in original_env.items():
                 os.environ[k] = v
     else:
         _do_run_file(cli, conf_file)
@@ -933,7 +939,7 @@ def _limit_to_str(limit):
 
 def _burst_to_str(burst):
     # no output if max_burst is not set
-    if len(burst.values()) == 0 or burst.values()[0] == 0:
+    if len(burst) == 0 or list(burst.values())[0] == 0:
         return ''
 
     if 'count' in burst:
@@ -1044,10 +1050,10 @@ def check_constraints(cli):
 
 
 def _show_tc_list(cli, tcs):
-    wids = sorted(list(set(map(lambda tc: getattr(tc, 'class').wid, tcs))))
+    wids = sorted(list(set([getattr(tc, 'class').wid for tc in tcs])))
 
     for wid in wids:
-        matched = filter(lambda tc: getattr(tc, 'class').wid == wid, tcs)
+        matched = [tc for tc in tcs if getattr(tc, 'class').wid == wid]
 
         root = _build_tcs_tree(matched)
         if wid == -1:
@@ -1151,7 +1157,7 @@ def _draw_pipeline(cli, field, units, last_stats=None):
                              stderr=subprocess.PIPE)
 
         for m in modules:
-            print >> f.stdin, '[%s]' % node_labels[m.name]
+            print('[%s]' % node_labels[m.name], file=f.stdin)
 
         for name in names:
             gates = cli.bess.get_module_info(name).ogates
@@ -1178,10 +1184,10 @@ def _draw_pipeline(cli, field, units, last_stats=None):
                 edge_attr = '{label::%d  %s %s %d:;}' % (
                     gate.ogate, label, units, gate.igate)
 
-                print >> f.stdin, '[%s] ->%s [%s]' % (
+                print('[%s] ->%s [%s]' % (
                     node_labels[name],
                     edge_attr,
-                    node_labels[gate.name])
+                    node_labels[gate.name]), file=f.stdin)
         output, error = f.communicate()
         f.wait()
         return output
@@ -1558,8 +1564,8 @@ def get_total(arr):
 
             if len(ports) > 1:
                 print_delta('Total', get_delta(
-                    get_total(last.values()),
-                    get_total(now.values())))
+                    get_total(list(last.values())),
+                    get_total(list(now.values()))))
 
             for port in ports:
                 last[port] = now[port]
@@ -1686,7 +1692,7 @@ def tcpdump_module(cli, module_name, direction, gate, opts):
         direction = 'out'
 
     fifo = tempfile.mktemp()
-    os.mkfifo(fifo, 0600)   # random people should not see packets...
+    os.mkfifo(fifo, 0o600)   # random people should not see packets...
 
     fd = os.open(fifo, os.O_RDWR)
 

bessctl/conf/metadata/attr_match.bess

@@ -19,5 +19,5 @@ em:1 -> Sink()
 em:2 -> Sink()
 
 # NOTE: metadata attribute values are stored in host order (little endian)!
-em.add(fields=['\xcc', '\x22\x11'], gate=1)
-em.add(fields=['\x42', '\x33\x44'], gate=2)
+em.add(fields=[b'\xcc', b'\x22\x11'], gate=1)
+em.add(fields=[b'\x42', b'\x33\x44'], gate=2)

bessctl/conf/perftest/bpf.bess

@@ -10,7 +10,7 @@ bpf:2 -> Sink()      # for matched packets (by the second filter)
 
 def run_testcase(exps, test_pkts):
     rewrite.clear()
-    rewrite.add(templates=map(lambda x: str(x), test_pkts))
+    rewrite.add(templates=map(lambda x: bytes(x), test_pkts))
 
     # the higher number, the higher priority.
     bpf.clear()
@@ -34,10 +34,10 @@ def run_testcase(exps, test_pkts):
 
     pps_total = pps_matched + pps_unmatched
 
-    print 'Total: %8.3fMpps   Matched: %8.3fMpps   Unmatched: %8.3fMpps' %  \
-            (pps_total / 1000000.0,
-             pps_matched / 1000000.0,
-             pps_unmatched / 1000000.0)
+    print('Total: %8.3fMpps   Matched: %8.3fMpps   Unmatched: %8.3fMpps' %
+          (pps_total / 1000000.0,
+           pps_matched / 1000000.0,
+           pps_unmatched / 1000000.0))
 
     bess.pause_all()
 
@@ -62,11 +62,11 @@ testcases = [
 ]
 
 for i, case in enumerate(testcases):
-    print
-    print 'Testcase %d: %s' % (i, ', '.join(case[0]))
+    print()
+    print('Testcase %d: %s' % (i, ', '.join(case[0])))
     pprint.pprint(case[1])
 
 print
 for i, case in enumerate(testcases):
-    print 'Testcase %d:\t\t' % i,
+    print('Testcase %d:\t\t' % i, end=' ')
     run_testcase(*case)

bessctl/conf/perftest/flowgen.bess

@@ -23,7 +23,7 @@ ip = scapy.IP(src=src_ip, dst=dst_ip)
 src_port = int($BESS_SRC_PORT!'10001')
 tcp = scapy.TCP(sport=src_port, dport=12345, seq=12345)
 payload = "BESS is the Queen of Packet Processing."
-pkt_template = str(eth/ip/tcp/payload)
+pkt_template = bytes(eth/ip/tcp/payload)
 
 # This script is multi threaded but connects to a single port.
 myport = PMDPort(port_id=0, num_inc_q=num_cores, num_out_q=num_cores)
@@ -49,7 +49,7 @@ while True:
     portstats = myport.get_port_stats().inc.packets
     delta = portstats - prev_portstats
     nextround = max(1e6, 1.1 * delta / sleeptime)
-    print("Received " + str(delta / sleeptime) + " pps. Ramping up to: " + str(nextround))
+    print("Received " + bytes(delta / sleeptime) + " pps. Ramping up to: " + bytes(nextround))
     for wid, pkt_src in flowgens.items():
         bess.pause_worker(wid)
         pkt_src.update(pps = nextround / num_cores)

bessctl/conf/perftest/loopback_vport.bess

@@ -39,32 +39,32 @@ def measure():
                     old_stats[i].inc.packets
         inc_mpps.append(pkts_diff / time_diff / 1000000.0)
 
-    print '%-15s' % 'CPU',
-    print '%7s' % '',
+    print('%-15s' % 'CPU', end=' ')
+    print('%7s' % '', end=' ')
     for i in range(len(ports)):
-        print '%7d' % cpu_set[i],
+        print('%7d' % cpu_set[i], end=' ')
     print
 
-    print '%-15s' % 'Ports',
-    print '%7s' % '(total)',
+    print('%-15s' % 'Ports', end=' ')
+    print('%7s' % '(total)', end=' ')
     for i in range(len(ports)):
-        print '%7s' % ports[i].name,
-    print
+        print('%7s' % ports[i].name, end=' ')
+    print()
 
-    print '-' * (8 * (len(ports) + 3))
+    print('-' * (8 * (len(ports) + 3)))
 
-    print '%-15s' % 'Outgoing (Mpps)',
-    print '%7.3f' % sum(out_mpps),
+    print('%-15s' % 'Outgoing (Mpps)', end=' ')
+    print('%7.3f' % sum(out_mpps), end=' ')
     for i in range(len(ports)):
-        print '%7.3f' % out_mpps[i],
-    print
+        print('%7.3f' % out_mpps[i], end=' ')
+    print()
 
-    print '%-15s' % 'Incoming (Mpps)',
-    print '%7.3f' % sum(inc_mpps),
+    print('%-15s' % 'Incoming (Mpps)', end=' ')
+    print('%7.3f' % sum(inc_mpps), end=' ')
     for i in range(len(ports)):
-        print '%7.3f' % inc_mpps[i],
-    print
-    print
+        print('%7.3f' % inc_mpps[i], end=' ')
+    print()
+    print()
 
 for cpu in range(CORE_START, CORE_END, CORE_STEP):
     v = VPort(loopback=1, rxq_cpus=[cpu])

bessctl/conf/perftest/nat.bess

@@ -10,7 +10,7 @@ eth = scapy.Ether(src='02:1e:67:9f:4d:ac', dst='06:16:3e:1b:72:32')
 ip = scapy.IP(src='10.0.0.1', dst='192.168.1.1')
 udp = scapy.UDP(sport=10001, dport=10002)
 payload = 'helloworld'
-pkt_bytes = str(eth/ip/udp/payload)
+pkt_bytes = bytes(eth/ip/udp/payload)
 
 Source() \
     -> Rewrite(templates=[pkt_bytes]) \

bessctl/conf/perftest/pktgen.bess

@@ -21,7 +21,7 @@ def build_pkt(size):
     payload = ('hello' + '0123456789' * 200)[:size-len(eth/ip/udp)]
     pkt = eth/ip/udp/payload
     pkt.show()
-    return str(pkt)
+    return bytes(pkt)
 
 if imix:
     # https://en.wikipedia.org/wiki/Internet_Mix
@@ -45,7 +45,7 @@ ports = [PMDPort(port_id=i, num_inc_q=num_cores, num_out_q=num_cores) \
          for i in range(num_ports)]
 
 for i in range(num_cores):
-    print("starting up worker: " + str(i))
+    print("starting up worker: " + bytes(i))
     bess.add_worker(wid=i, core=i)
 
     src = Source()