Convert tests to Python 3

Fix #938

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ env:
 jobs:
   build:
     name: test
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     continue-on-error: ${{ matrix.allow_failure }}
     strategy:
       fail-fast: true
@@ -55,16 +55,16 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 2.7.17
+          python-version: 3.9
 
       - name: Install O/S packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y python-virtualenv graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+          sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
 
       - name: Install Python packages
         run: |
-          pip install six packaging appdirs
+          pip install six packaging appdirs virtualenv
           pip install -U pip setuptools
           pip install -r system/requirements.txt
 

Makefile

@@ -79,4 +79,3 @@ version:
 	@echo $(VERSION)
 
 .PHONY: man modules version release goxc
-

system/fs_endpoint_lib.py

@@ -38,11 +38,14 @@ def check_is_symlink(self, path):
         if not os.path.islink(os.path.join(os.environ["HOME"], ".aptly", path)):
             raise Exception("path %s is not a symlink" % (path, ))
 
+    def is_hardlink(self, path):
+        return os.stat(os.path.join(os.environ["HOME"], ".aptly", path)).st_nlink >= 2
+
     def check_is_hardlink(self, path):
-        if os.stat(os.path.join(os.environ["HOME"], ".aptly", path)) <= 1:
+        if not self.is_hardlink(path):
             raise Exception("path %s is not a hardlink" % (path, ))
 
     def check_is_copy(self, path):
         fullpath = os.path.join(os.environ["HOME"], ".aptly", path)
-        if not (os.path.isfile(fullpath) and not self.check_is_hardlink(path)):
+        if not (os.path.isfile(fullpath) and not self.is_hardlink(path)):
             raise Exception("path %s is not a copy" % (path, ))

system/lib.py

@@ -13,17 +13,27 @@
 import shutil
 import string
 import threading
-import urllib
+import urllib.error
+import urllib.parse
+import urllib.request
 import pprint
-import SocketServer
-import SimpleHTTPServer
+import socketserver
+import http.server
+import zlib
 
 
-class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+def ungzip_if_required(output):
+    if isinstance(output, bytes) and output.startswith(b"\x1f\x8b"):
+        return zlib.decompress(output, 16 + zlib.MAX_WBITS).decode('utf-8')
+
+    return output
+
+
+class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
     pass
 
 
-class FileHTTPServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+class FileHTTPServerRequestHandler(http.server.SimpleHTTPRequestHandler):
     def translate_path(self, path):
         """Translate a /-separated PATH to the local filename syntax.
 
@@ -35,9 +45,9 @@ def translate_path(self, path):
         # abandon query parameters
         path = path.split('?', 1)[0]
         path = path.split('#', 1)[0]
-        path = posixpath.normpath(urllib.unquote(path))
+        path = posixpath.normpath(urllib.parse.unquote(path))
         words = path.split('/')
-        words = filter(None, words)
+        words = [_f for _f in words if _f]
         path = self.rootPath
         for word in words:
             _, word = os.path.splitdrive(word)
@@ -70,7 +80,7 @@ def __init__(self):
     def find_gpg(self, executables, expected_version):
         for executable in executables:
             try:
-                output = subprocess.check_output([executable, "--version"])
+                output = subprocess.check_output([executable, "--version"], text=True)
                 if expected_version in output:
                     return executable
             except Exception:
@@ -90,7 +100,7 @@ def __init__(self):
     def find_dot(self, executables):
         for executable in executables:
             try:
-                subprocess.check_output([executable, "-V"])
+                subprocess.check_output([executable, "-V"], text=True)
                 return executable
             except Exception:
                 pass
@@ -114,6 +124,7 @@ class BaseTest(object):
     requiresGPG1 = False
     requiresGPG2 = False
     requiresDot = False
+    sortOutput = False
 
     expectedCode = 0
     configFile = {
@@ -198,12 +209,12 @@ def fixture_available(self):
 
     def prepare_fixture(self):
         if self.fixturePool:
-            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0755)
+            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0o755)
             os.symlink(self.fixturePoolDir, os.path.join(
                 os.environ["HOME"], ".aptly", "pool"))
 
         if self.fixturePoolCopy:
-            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0755)
+            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0o755)
             shutil.copytree(self.fixturePoolDir, os.path.join(
                 os.environ["HOME"], ".aptly", "pool"), ignore=shutil.ignore_patterns(".git"))
 
@@ -228,12 +239,17 @@ def prepare_fixture(self):
             for cmd in self.fixtureCmds:
                 self.run_cmd(cmd)
 
+    def sort_lines(self, output):
+        return "\n".join(sorted(self.ensure_utf8(output).split("\n")))
+
     def run(self):
-        self.output = self.output_processor(
-            self.run_cmd(self.runCmd, self.expectedCode))
+        output = self.run_cmd(self.runCmd, self.expectedCode)
+        if self.sortOutput:
+            output = self.sort_lines(output)
+        self.output = self.output_processor(output)
 
     def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None):
-        if not hasattr(command, "__iter__"):
+        if isinstance(command, str):
             params = {
                 'files': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"),
                 'changes': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes"),
@@ -245,12 +261,12 @@ def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None):
                 params['url'] = self.webServerUrl
 
             command = string.Template(command).substitute(params)
-
             command = shlex.split(command)
+
         environ = os.environ.copy()
         environ["LC_ALL"] = "C"
         environ.update(self.environmentOverride)
-        return subprocess.Popen(command, stderr=stderr, stdout=stdout, env=environ)
+        return subprocess.Popen(command, stderr=stderr, stdout=stdout, env=environ, group=True)
 
     def run_cmd(self, command, expected_code=0):
         try:
@@ -261,7 +277,7 @@ def run_cmd(self, command, expected_code=0):
                     raise Exception("exit code %d != %d (output: %s)" % (
                         proc.returncode, expected_code, output))
             return output
-        except Exception, e:
+        except Exception as e:
             raise Exception("Running command %s failed: %s" %
                             (command, str(e)))
 
@@ -315,19 +331,20 @@ def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code
             else:
                 raise
 
-    def read_file(self, path):
-        with open(os.path.join(os.environ["HOME"], ".aptly", path), "r") as f:
+    def read_file(self, path, mode=''):
+        with open(os.path.join(os.environ["HOME"], ".aptly", path), "r" + mode) as f:
             return f.read()
 
     def delete_file(self, path):
         os.unlink(os.path.join(os.environ["HOME"], ".aptly", path))
 
-    def check_file_contents(self, path, gold_name, match_prepare=None):
-        contents = self.read_file(path)
+    def check_file_contents(self, path, gold_name, match_prepare=None, mode='', ensure_utf8=True):
+        contents = self.read_file(path, mode=mode)
         try:
 
             self.verify_match(self.get_gold(gold_name),
-                              contents, match_prepare=match_prepare)
+                              contents, match_prepare=match_prepare,
+                              ensure_utf8=ensure_utf8)
         except:  # noqa: E722
             if self.captureResults:
                 if match_prepare is not None:
@@ -376,9 +393,13 @@ def check_in(self, item, l):
         if item not in l:
             raise Exception("item %r not in %r", item, l)
 
+    def check_not_in(self, item, l):
+        if item in l:
+            raise Exception("item %r in %r", item, l)
+
     def check_subset(self, a, b):
         diff = ''
-        for k, v in a.items():
+        for k, v in list(a.items()):
             if k not in b:
                 diff += "unexpected key '%s'\n" % (k,)
             elif b[k] != v:
@@ -387,7 +408,16 @@ def check_subset(self, a, b):
         if diff:
             raise Exception("content doesn't match:\n" + diff)
 
-    def verify_match(self, a, b, match_prepare=None):
+    def ensure_utf8(self, a):
+        if isinstance(a, bytes):
+            return a.decode('utf-8')
+        return a
+
+    def verify_match(self, a, b, match_prepare=None, ensure_utf8=True):
+        if ensure_utf8:
+            a = self.ensure_utf8(a)
+            b = self.ensure_utf8(b)
+
         if match_prepare is not None:
             a = match_prepare(a)
             b = match_prepare(b)

system/run.py

@@ -24,6 +24,9 @@ def colored(s, **kwargs):
         return s
 
 
+PYTHON_MINIMUM_VERSION = (3, 9)
+
+
 def natural_key(string_):
     """See https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/"""
     return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)]
@@ -107,18 +110,18 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
     if lastBase is not None:
         lastBase.shutdown_class()
 
-    print "TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % (
-        numTests, numTests - numFailed, numFailed, numSkipped)
+    print("TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % (
+        numTests, numTests - numFailed, numFailed, numSkipped))
 
     if len(fails) > 0:
-        print "\nFAILURES (%d):" % (len(fails), )
+        print("\nFAILURES (%d):" % (len(fails), ))
 
         for (test, t, typ, val, tb, testModule) in fails:
             doc = t.__doc__ or ''
-            print "%s:%s %s" % (test, t.__class__.__name__,
-                                testModule.__name__ + ": " + doc.strip())
+            print("%s:%s %s" % (test, t.__class__.__name__,
+                                testModule.__name__ + ": " + doc.strip()))
             traceback.print_exception(typ, val, tb)
-            print "=" * 60
+            print("=" * 60)
 
         sys.exit(1)
 
@@ -128,14 +131,17 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
         try:
             os.environ['APTLY_VERSION'] = os.popen(
                 "make version").read().strip()
-        except BaseException, e:
-            print "Failed to capture current version: ", e
+        except BaseException as e:
+            print("Failed to capture current version: ", e)
+
+    if sys.version_info < PYTHON_MINIMUM_VERSION:
+        raise RuntimeError(f'Tests require Python {PYTHON_MINIMUM_VERSION} or higher.')
 
-    output = subprocess.check_output(['gpg1', '--version'])
+    output = subprocess.check_output(['gpg1', '--version'], text=True)
     if not output.startswith('gpg (GnuPG) 1'):
         raise RuntimeError('Tests require gpg v1')
 
-    output = subprocess.check_output(['gpgv1', '--version'])
+    output = subprocess.check_output(['gpgv1', '--version'], text=True)
     if not output.startswith('gpgv (GnuPG) 1'):
         raise RuntimeError('Tests require gpgv v1')
 

system/s3_lib.py

@@ -8,10 +8,10 @@
     if 'AWS_SECRET_ACCESS_KEY' in os.environ and 'AWS_ACCESS_KEY_ID' in os.environ:
         s3_conn = boto.connect_s3()
     else:
-        print "S3 tests disabled: AWS creds not found in the environment"
+        print("S3 tests disabled: AWS creds not found in the environment")
         s3_conn = None
-except ImportError, e:
-    print "S3 tests disabled: can't import boto", e
+except ImportError as e:
+    print("S3 tests disabled: can't import boto", e)
     s3_conn = None
 
 

system/swift_lib.py

@@ -25,10 +25,10 @@
         swift_conn = swiftclient.Connection(auth_url, auth_username,
                                             auth_password, auth_version=1)
     else:
-        print "Swift tests disabled: OpenStack creds not found in the environment"
+        print("Swift tests disabled: OpenStack creds not found in the environment")
         swift_conn = None
-except ImportError, e:
-    print "Swift tests disabled: unable to import swiftclient", e
+except ImportError as e:
+    print("Swift tests disabled: unable to import swiftclient", e)
     swift_conn = None