Upgrade to JRE17 and Upgrade System VMs/VRs to Python3 and Debian 12

systemvm/agent/noVNC/keymaps/generate-language-keymaps.py

@@ -95,7 +95,7 @@ def generate_js_file(keymap_file):
     js_config.append(" * layout     : %s\n" % layout)
     js_config.append(" */\n")
     js_config.append("export default {\n")
-    for keycode in dict(sorted(result_mappings.items(), key=lambda item: int(item[0]))):
+    for keycode in dict(sorted(list(result_mappings.items()), key=lambda item: int(item[0]))):
         js_config.append("%10s : \"%s\",\n" % ("\"" + str(keycode) + "\"", result_mappings[keycode].strip()))
     js_config.append("}\n")
     for line in js_config:

systemvm/debian/opt/cloud/bin/configure.py

@@ -21,8 +21,8 @@
 import os
 import re
 import sys
-import urllib
-import urllib2
+import urllib.request, urllib.parse, urllib.error
+import urllib.request, urllib.error, urllib.parse
 import time
 import copy
 
@@ -107,10 +107,10 @@ def __update(self, vm_ip, password):
             if proc.find():
                 url = "http://%s:8080/" % server_ip
                 payload = {"ip": vm_ip, "password": password, "token": token}
-                data = urllib.urlencode(payload)
-                request = urllib2.Request(url, data=data, headers={"DomU_Request": "save_password"})
+                data = urllib.parse.urlencode(payload)
+                request = urllib.request.Request(url, data=data, headers={"DomU_Request": "save_password"})
                 try:
-                    resp = urllib2.urlopen(request, data)
+                    resp = urllib.request.urlopen(request, data)
                     logging.debug("Update password server result: http:%s, content:%s" % (resp.code, resp.read()))
                 except Exception as e:
                     logging.error("Failed to update password server due to: %s" % e)
@@ -165,15 +165,15 @@ def add_rule(self):
             icmp_type = ''
             rule = self.rule
             icmp_type = "any"
-            if "icmp_type" in self.rule.keys() and self.rule['icmp_type'] != -1:
+            if "icmp_type" in list(self.rule.keys()) and self.rule['icmp_type'] != -1:
                 icmp_type = self.rule['icmp_type']
-            if "icmp_code" in self.rule.keys() and rule['icmp_code'] != -1:
+            if "icmp_code" in list(self.rule.keys()) and rule['icmp_code'] != -1:
                 icmp_type = "%s/%s" % (self.rule['icmp_type'], self.rule['icmp_code'])
             rnge = ''
-            if "first_port" in self.rule.keys() and \
+            if "first_port" in list(self.rule.keys()) and \
                self.rule['first_port'] == self.rule['last_port']:
                 rnge = " --dport %s " % self.rule['first_port']
-            if "first_port" in self.rule.keys() and \
+            if "first_port" in list(self.rule.keys()) and \
                self.rule['first_port'] != self.rule['last_port']:
                 rnge = " --dport %s:%s" % (rule['first_port'], rule['last_port'])
 
@@ -278,14 +278,14 @@ def __init__(self, obj, config):
             self.device = obj['device']
             self.ip = obj['nic_ip']
             self.ip6_cidr = None
-            if "nic_ip6_cidr" in obj.keys():
+            if "nic_ip6_cidr" in list(obj.keys()):
                 self.ip6_cidr = obj['nic_ip6_cidr']
             self.netmask = obj['nic_netmask']
             self.config = config
             self.cidr = "%s/%s" % (self.ip, self.netmask)
-            if "ingress_rules" in obj.keys():
+            if "ingress_rules" in list(obj.keys()):
                 self.ingress = obj['ingress_rules']
-            if "egress_rules" in obj.keys():
+            if "egress_rules" in list(obj.keys()):
                 self.egress = obj['egress_rules']
             self.fw = config.get_fw()
             self.ipv6_acl = config.get_ipv6_acl()
@@ -352,7 +352,7 @@ def __process_ip6(self, direction, rule_list):
                         proto = "%s dport %s" % (proto, port)
 
                 action = "drop"
-                if 'allowed' in rule.keys() and rule['allowed']:
+                if 'allowed' in list(rule.keys()) and rule['allowed']:
                     action = "accept"
 
                 rstr = addr
@@ -411,21 +411,21 @@ def init_vpc(self, direction, acl, rule, config):
                 self.type = rule['type']
                 self.icmp_type = "any"
                 self.protocol = self.type
-                if "icmp_type" in rule.keys() and rule['icmp_type'] != -1:
+                if "icmp_type" in list(rule.keys()) and rule['icmp_type'] != -1:
                     self.icmp_type = rule['icmp_type']
-                if "icmp_code" in rule.keys() and rule['icmp_code'] != -1:
+                if "icmp_code" in list(rule.keys()) and rule['icmp_code'] != -1:
                     self.icmp_type = "%s/%s" % (self.icmp_type, rule['icmp_code'])
                 if self.type == "protocol":
                     if rule['protocol'] == 41:
                         rule['protocol'] = "ipv6"
                     self.protocol = rule['protocol']
                 self.action = "DROP"
                 self.dport = ""
-                if 'allowed' in rule.keys() and rule['allowed']:
+                if 'allowed' in list(rule.keys()) and rule['allowed']:
                     self.action = "ACCEPT"
-                if 'first_port' in rule.keys():
+                if 'first_port' in list(rule.keys()):
                     self.dport = "-m %s --dport %s" % (self.protocol, rule['first_port'])
-                if 'last_port' in rule.keys() and self.dport and \
+                if 'last_port' in list(rule.keys()) and self.dport and \
                    rule['last_port'] != rule['first_port']:
                     self.dport = "%s:%s" % (self.dport, rule['last_port'])
 
@@ -640,23 +640,26 @@ def __createfile(self, ip, folder, file, data):
         fh = open(dest, "w")
         self.__exflock(fh)
         if data is not None:
-            fh.write(data)
+            if type(data) == str:
+                fh.write(data)
+            elif type(data) == bytes:
+                fh.write(data.decode())
         else:
             fh.write("")
         self.__unflock(fh)
         fh.close()
-        os.chmod(dest, 0644)
+        os.chmod(dest, 0o644)
 
         if folder == "metadata" or folder == "meta-data":
             try:
-                os.makedirs(metamanifestdir, 0755)
+                os.makedirs(metamanifestdir, 0o755)
             except OSError as e:
                 # error 17 is already exists, we do it this way for concurrency
                 if e.errno != 17:
-                    print "failed to make directories " + metamanifestdir + " due to :" + e.strerror
+                    print("failed to make directories " + metamanifestdir + " due to :" + e.strerror)
                     sys.exit(1)
             if os.path.exists(metamanifest):
-                fh = open(metamanifest, "r+a")
+                fh = open(metamanifest, "a+")
                 self.__exflock(fh)
                 if file not in fh.read():
                     fh.write(file + '\n')
@@ -670,17 +673,17 @@ def __createfile(self, ip, folder, file, data):
                 fh.close()
 
         if os.path.exists(metamanifest):
-            os.chmod(metamanifest, 0644)
+            os.chmod(metamanifest, 0o644)
 
     def __htaccess(self, ip, folder, file):
         entry = "RewriteRule ^" + file + "$  ../" + folder + "/%{REMOTE_ADDR}/" + file + " [L,NC,QSA]"
         htaccessFolder = "/var/www/html/latest"
         htaccessFile = htaccessFolder + "/.htaccess"
 
-        CsHelper.mkdir(htaccessFolder, 0755, True)
+        CsHelper.mkdir(htaccessFolder, 0o755, True)
 
         if os.path.exists(htaccessFile):
-            fh = open(htaccessFile, "r+a")
+            fh = open(htaccessFile, "a+")
             self.__exflock(fh)
             if entry not in fh.read():
                 fh.write(entry + '\n')
@@ -699,11 +702,11 @@ def __htaccess(self, ip, folder, file):
         htaccessFile = htaccessFolder+"/.htaccess"
 
         try:
-            os.makedirs(htaccessFolder, 0755)
+            os.makedirs(htaccessFolder, 0o755)
         except OSError as e:
             # error 17 is already exists, we do it this way for sake of concurrency
             if e.errno != 17:
-                print "failed to make directories " + htaccessFolder + " due to :" + e.strerror
+                print("failed to make directories " + htaccessFolder + " due to :" + e.strerror)
                 sys.exit(1)
 
         fh = open(htaccessFile, "w")
@@ -717,7 +720,7 @@ def __htaccess(self, ip, folder, file):
             htaccessFolder = "/var/www/html/latest"
             htaccessFile = htaccessFolder + "/.htaccess"
 
-            fh = open(htaccessFile, "r+a")
+            fh = open(htaccessFile, "a+")
             self.__exflock(fh)
             if entry not in fh.read():
                 fh.write(entry + '\n')
@@ -734,15 +737,15 @@ def __exflock(self, file):
         try:
             flock(file, LOCK_EX)
         except IOError as e:
-            print "failed to lock file" + file.name + " due to : " + e.strerror
+            print("failed to lock file" + file.name + " due to : " + e.strerror)
             sys.exit(1)  # FIXME
         return True
 
     def __unflock(self, file):
         try:
             flock(file, LOCK_UN)
         except IOError as e:
-            print "failed to unlock file" + file.name + " due to : " + e.strerror
+            print("failed to unlock file" + file.name + " due to : " + e.strerror)
             sys.exit(1)  # FIXME
         return True
 
@@ -868,9 +871,9 @@ def configure_ipsec(self, obj):
 
         # This will load the new config
         CsHelper.execute("ipsec reload")
-        os.chmod(vpnsecretsfile, 0400)
+        os.chmod(vpnsecretsfile, 0o400)
 
-        for i in xrange(3):
+        for i in range(3):
             done = True
             for peeridx in range(0, len(peerlistarr)):
                 # Check for the proper connection and subnet
@@ -1383,7 +1386,7 @@ def main(argv):
         databag_map.pop("guest_network")
 
     def execDatabag(key, db):
-        if key not in db.keys() or 'executor' not in db[key]:
+        if key not in list(db.keys()) or 'executor' not in db[key]:
             logging.warn("Unable to find config or executor(s) for the databag type %s" % key)
             return
         for executor in db[key]['executor']:
@@ -1397,10 +1400,10 @@ def execIptables(config):
 
     if json_type == "cmd_line":
         logging.debug("cmd_line.json changed. All other files will be processed as well.")
-        for key in databag_map.keys():
+        for key in list(databag_map.keys()):
             execDatabag(key, databag_map)
         execIptables(config)
-    elif json_type in databag_map.keys():
+    elif json_type in list(databag_map.keys()):
         execDatabag(json_type, databag_map)
         if databag_map[json_type]['process_iptables']:
             execIptables(config)

systemvm/debian/opt/cloud/bin/cs/CsAddress.py

@@ -19,11 +19,11 @@
 from netaddr import IPAddress, IPNetwork
 import subprocess
 import time
-import CsHelper
-from CsDatabag import CsDataBag
-from CsApp import CsApache, CsDnsmasq, CsPasswdSvc
-from CsRoute import CsRoute
-from CsRule import CsRule
+from . import CsHelper
+from .CsDatabag import CsDataBag
+from .CsApp import CsApache, CsDnsmasq, CsPasswdSvc
+from .CsRoute import CsRoute
+from .CsRule import CsRule
 
 VRRP_TYPES = ['guest']
 
@@ -556,7 +556,7 @@ def fw_vpcrouter(self):
                                 "-A POSTROUTING -o %s -j SNAT --to-source %s" %
                                 (self.dev, self.address['public_ip'])])
             if self.get_gateway() == self.get_ip_address():
-                for inf, addresses in self.config.address().dbag.iteritems():
+                for inf, addresses in self.config.address().dbag.items():
                     if not inf.startswith("eth"):
                         continue
                     for address in addresses:
@@ -625,7 +625,7 @@ def post_config_change(self, method):
             if self.config.is_vpc():
                 if self.get_type() in ["public"] and "gateway" in self.address and self.address["gateway"] and self.address["gateway"] != "None":
                     route.add_route(self.dev, self.address["gateway"])
-                    for inf, addresses in self.config.address().dbag.iteritems():
+                    for inf, addresses in self.config.address().dbag.items():
                         if not inf.startswith("eth"):
                             continue
                         for address in addresses:
@@ -706,7 +706,7 @@ def list(self):
                 self.iplist[cidr] = self.dev
 
     def configured(self):
-        if self.address['cidr'] in self.iplist.keys():
+        if self.address['cidr'] in list(self.iplist.keys()):
             return True
         return False
 
@@ -735,7 +735,7 @@ def getDevice(self):
         return self.dev
 
     def hasIP(self, ip):
-        return ip in self.address.values()
+        return ip in list(self.address.values())
 
     def arpPing(self):
         cmd = "arping -c 1 -I %s -A -U -s %s %s" % (
@@ -746,7 +746,7 @@ def arpPing(self):
 
     # Delete any ips that are configured but not in the bag
     def compare(self, bag):
-        if len(self.iplist) > 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0):
+        if len(self.iplist) > 0 and (self.dev not in list(bag.keys()) or len(bag[self.dev]) == 0):
             # Remove all IPs on this device
             logging.info(
                 "Will remove all configured addresses on device %s", self.dev)
@@ -757,13 +757,13 @@ def compare(self, bag):
         # This condition should not really happen but did :)
         # It means an apache file got orphaned after a guest network address
         # was deleted
-        if len(self.iplist) == 0 and (self.dev not in bag.keys() or len(bag[self.dev]) == 0):
+        if len(self.iplist) == 0 and (self.dev not in list(bag.keys()) or len(bag[self.dev]) == 0):
             app = CsApache(self)
             app.remove()
 
         for ip in self.iplist:
             found = False
-            if self.dev in bag.keys():
+            if self.dev in list(bag.keys()):
                 for address in bag[self.dev]:
                     self.setAddress(address)
                     if (self.hasIP(ip) or self.is_guest_gateway(address, ip)) and address["add"]:
@@ -796,7 +796,7 @@ def delete(self, ip):
         remove = []
         if ip == "all":
             logging.info("Removing addresses from device %s", self.dev)
-            remove = self.iplist.keys()
+            remove = list(self.iplist.keys())
         else:
             remove.append(ip)
         for ip in remove:

systemvm/debian/opt/cloud/bin/cs/CsApp.py

@@ -16,8 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 import os
-from CsFile import CsFile
-import CsHelper
+from .CsFile import CsFile
+from . import CsHelper
 
 
 class CsApp:

systemvm/debian/opt/cloud/bin/cs/CsConfig.py

@@ -16,8 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from CsDatabag import CsCmdLine, CsGuestNetwork
-from CsAddress import CsAddress
+from .CsDatabag import CsCmdLine, CsGuestNetwork
+from .CsAddress import CsAddress
 import logging
 
 

systemvm/debian/opt/cloud/bin/cs/CsDatabag.py

@@ -33,7 +33,7 @@ def __init__(self, key, config=None):
             self.config = config
 
     def dump(self):
-        print self.dbag
+        print(self.dbag)
 
     def get_bag(self):
         return self.dbag
@@ -217,7 +217,7 @@ def get_router_ip6prelen(self, devname=None):
         if devname:
             return self.__get_device_router_ip6prelen(devname)
         else:
-            for key in self.dbag.keys():
+            for key in list(self.dbag.keys()):
                 ip6prelen = self.__get_device_router_ip6prelen(key)
                 if ip6prelen:
                     return ip6prelen
@@ -234,7 +234,7 @@ def get_router_ip6gateway(self, devname=None):
         if devname:
             return self.__get_device_router_ip6gateway(devname)
         else:
-            for key in self.dbag.keys():
+            for key in list(self.dbag.keys()):
                 ip6gateway = self.__get_device_router_ip6gateway(key)
                 if ip6gateway:
                     return ip6gateway

systemvm/debian/opt/cloud/bin/cs/CsDhcp.py

@@ -14,13 +14,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import CsHelper
+from . import CsHelper
 import logging
 import os
 from netaddr import *
 from random import randint
 import json
-from CsGuestNetwork import CsGuestNetwork
+from .CsGuestNetwork import CsGuestNetwork
 from cs.CsDatabag import CsDataBag
 from cs.CsFile import CsFile
 from cs.CsAddress import CsIP