diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8eb6f889b..f8551f23d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ env: jobs: build: name: test - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 continue-on-error: ${{ matrix.allow_failure }} strategy: fail-fast: true @@ -55,16 +55,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 2.7.17 + python-version: 3.9 - name: Install O/S packages run: | sudo apt-get update - sudo apt-get install -y python-virtualenv graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make + sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make - name: Install Python packages run: | - pip install six packaging appdirs + pip install six packaging appdirs virtualenv pip install -U pip setuptools pip install -r system/requirements.txt diff --git a/Makefile b/Makefile index de5093698..80d951f67 100644 --- a/Makefile +++ b/Makefile @@ -79,4 +79,3 @@ version: @echo $(VERSION) .PHONY: man modules version release goxc - diff --git a/system/fs_endpoint_lib.py b/system/fs_endpoint_lib.py index dcb4e803f..87a191beb 100644 --- a/system/fs_endpoint_lib.py +++ b/system/fs_endpoint_lib.py @@ -38,11 +38,14 @@ def check_is_symlink(self, path): if not os.path.islink(os.path.join(os.environ["HOME"], ".aptly", path)): raise Exception("path %s is not a symlink" % (path, )) + def is_hardlink(self, path): + return os.stat(os.path.join(os.environ["HOME"], ".aptly", path)).st_nlink >= 2 + def check_is_hardlink(self, path): - if os.stat(os.path.join(os.environ["HOME"], ".aptly", path)) <= 1: + if not self.is_hardlink(path): raise Exception("path %s is not a hardlink" % (path, )) def check_is_copy(self, path): fullpath = os.path.join(os.environ["HOME"], ".aptly", path) - if not (os.path.isfile(fullpath) and not self.check_is_hardlink(path)): + if not (os.path.isfile(fullpath) and not self.is_hardlink(path)): raise Exception("path %s is not a copy" % (path, )) diff --git a/system/lib.py b/system/lib.py index b9acbdbe8..9c8ea6b02 100644 --- a/system/lib.py +++ b/system/lib.py @@ -13,17 +13,27 @@ import shutil import string import threading -import urllib +import urllib.error +import urllib.parse +import urllib.request import pprint -import SocketServer -import SimpleHTTPServer +import socketserver +import http.server +import zlib -class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): +def ungzip_if_required(output): + if isinstance(output, bytes) and output.startswith(b"\x1f\x8b"): + return zlib.decompress(output, 16 + zlib.MAX_WBITS).decode('utf-8') + + return output + + +class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): pass -class FileHTTPServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class FileHTTPServerRequestHandler(http.server.SimpleHTTPRequestHandler): def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. @@ -35,9 +45,9 @@ def translate_path(self, path): # abandon query parameters path = path.split('?', 1)[0] path = path.split('#', 1)[0] - path = posixpath.normpath(urllib.unquote(path)) + path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') - words = filter(None, words) + words = [_f for _f in words if _f] path = self.rootPath for word in words: _, word = os.path.splitdrive(word) @@ -70,7 +80,7 @@ def __init__(self): def find_gpg(self, executables, expected_version): for executable in executables: try: - output = subprocess.check_output([executable, "--version"]) + output = subprocess.check_output([executable, "--version"], text=True) if expected_version in output: return executable except Exception: @@ -90,7 +100,7 @@ def __init__(self): def find_dot(self, executables): for executable in executables: try: - subprocess.check_output([executable, "-V"]) + subprocess.check_output([executable, "-V"], text=True) return executable except Exception: pass @@ -114,6 +124,7 @@ class BaseTest(object): requiresGPG1 = False requiresGPG2 = False requiresDot = False + sortOutput = False expectedCode = 0 configFile = { @@ -198,12 +209,12 @@ def fixture_available(self): def prepare_fixture(self): if self.fixturePool: - os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0755) + os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0o755) os.symlink(self.fixturePoolDir, os.path.join( os.environ["HOME"], ".aptly", "pool")) if self.fixturePoolCopy: - os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0755) + os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0o755) shutil.copytree(self.fixturePoolDir, os.path.join( os.environ["HOME"], ".aptly", "pool"), ignore=shutil.ignore_patterns(".git")) @@ -228,12 +239,17 @@ def prepare_fixture(self): for cmd in self.fixtureCmds: self.run_cmd(cmd) + def sort_lines(self, output): + return "\n".join(sorted(self.ensure_utf8(output).split("\n"))) + def run(self): - self.output = self.output_processor( - self.run_cmd(self.runCmd, self.expectedCode)) + output = self.run_cmd(self.runCmd, self.expectedCode) + if self.sortOutput: + output = self.sort_lines(output) + self.output = self.output_processor(output) def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None): - if not hasattr(command, "__iter__"): + if isinstance(command, str): params = { 'files': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"), 'changes': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes"), @@ -245,12 +261,12 @@ def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None): params['url'] = self.webServerUrl command = string.Template(command).substitute(params) - command = shlex.split(command) + environ = os.environ.copy() environ["LC_ALL"] = "C" environ.update(self.environmentOverride) - return subprocess.Popen(command, stderr=stderr, stdout=stdout, env=environ) + return subprocess.Popen(command, stderr=stderr, stdout=stdout, env=environ, group=True) def run_cmd(self, command, expected_code=0): try: @@ -261,7 +277,7 @@ def run_cmd(self, command, expected_code=0): raise Exception("exit code %d != %d (output: %s)" % ( proc.returncode, expected_code, output)) return output - except Exception, e: + except Exception as e: raise Exception("Running command %s failed: %s" % (command, str(e))) @@ -315,19 +331,20 @@ def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code else: raise - def read_file(self, path): - with open(os.path.join(os.environ["HOME"], ".aptly", path), "r") as f: + def read_file(self, path, mode=''): + with open(os.path.join(os.environ["HOME"], ".aptly", path), "r" + mode) as f: return f.read() def delete_file(self, path): os.unlink(os.path.join(os.environ["HOME"], ".aptly", path)) - def check_file_contents(self, path, gold_name, match_prepare=None): - contents = self.read_file(path) + def check_file_contents(self, path, gold_name, match_prepare=None, mode='', ensure_utf8=True): + contents = self.read_file(path, mode=mode) try: self.verify_match(self.get_gold(gold_name), - contents, match_prepare=match_prepare) + contents, match_prepare=match_prepare, + ensure_utf8=ensure_utf8) except: # noqa: E722 if self.captureResults: if match_prepare is not None: @@ -376,9 +393,13 @@ def check_in(self, item, l): if item not in l: raise Exception("item %r not in %r", item, l) + def check_not_in(self, item, l): + if item in l: + raise Exception("item %r in %r", item, l) + def check_subset(self, a, b): diff = '' - for k, v in a.items(): + for k, v in list(a.items()): if k not in b: diff += "unexpected key '%s'\n" % (k,) elif b[k] != v: @@ -387,7 +408,16 @@ def check_subset(self, a, b): if diff: raise Exception("content doesn't match:\n" + diff) - def verify_match(self, a, b, match_prepare=None): + def ensure_utf8(self, a): + if isinstance(a, bytes): + return a.decode('utf-8') + return a + + def verify_match(self, a, b, match_prepare=None, ensure_utf8=True): + if ensure_utf8: + a = self.ensure_utf8(a) + b = self.ensure_utf8(b) + if match_prepare is not None: a = match_prepare(a) b = match_prepare(b) diff --git a/system/run.py b/system/run.py index 2cf4f39a3..e27377d08 100755 --- a/system/run.py +++ b/system/run.py @@ -24,6 +24,9 @@ def colored(s, **kwargs): return s +PYTHON_MINIMUM_VERSION = (3, 9) + + def natural_key(string_): """See https://blog.codinghorror.com/sorting-for-humans-natural-sort-order/""" return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] @@ -107,18 +110,18 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non if lastBase is not None: lastBase.shutdown_class() - print "TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % ( - numTests, numTests - numFailed, numFailed, numSkipped) + print("TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % ( + numTests, numTests - numFailed, numFailed, numSkipped)) if len(fails) > 0: - print "\nFAILURES (%d):" % (len(fails), ) + print("\nFAILURES (%d):" % (len(fails), )) for (test, t, typ, val, tb, testModule) in fails: doc = t.__doc__ or '' - print "%s:%s %s" % (test, t.__class__.__name__, - testModule.__name__ + ": " + doc.strip()) + print("%s:%s %s" % (test, t.__class__.__name__, + testModule.__name__ + ": " + doc.strip())) traceback.print_exception(typ, val, tb) - print "=" * 60 + print("=" * 60) sys.exit(1) @@ -128,14 +131,17 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non try: os.environ['APTLY_VERSION'] = os.popen( "make version").read().strip() - except BaseException, e: - print "Failed to capture current version: ", e + except BaseException as e: + print("Failed to capture current version: ", e) + + if sys.version_info < PYTHON_MINIMUM_VERSION: + raise RuntimeError(f'Tests require Python {PYTHON_MINIMUM_VERSION} or higher.') - output = subprocess.check_output(['gpg1', '--version']) + output = subprocess.check_output(['gpg1', '--version'], text=True) if not output.startswith('gpg (GnuPG) 1'): raise RuntimeError('Tests require gpg v1') - output = subprocess.check_output(['gpgv1', '--version']) + output = subprocess.check_output(['gpgv1', '--version'], text=True) if not output.startswith('gpgv (GnuPG) 1'): raise RuntimeError('Tests require gpgv v1') diff --git a/system/s3_lib.py b/system/s3_lib.py index bf74a93b7..9940fe136 100644 --- a/system/s3_lib.py +++ b/system/s3_lib.py @@ -8,10 +8,10 @@ if 'AWS_SECRET_ACCESS_KEY' in os.environ and 'AWS_ACCESS_KEY_ID' in os.environ: s3_conn = boto.connect_s3() else: - print "S3 tests disabled: AWS creds not found in the environment" + print("S3 tests disabled: AWS creds not found in the environment") s3_conn = None -except ImportError, e: - print "S3 tests disabled: can't import boto", e +except ImportError as e: + print("S3 tests disabled: can't import boto", e) s3_conn = None diff --git a/system/swift_lib.py b/system/swift_lib.py index 8457276e3..691611bec 100644 --- a/system/swift_lib.py +++ b/system/swift_lib.py @@ -25,10 +25,10 @@ swift_conn = swiftclient.Connection(auth_url, auth_username, auth_password, auth_version=1) else: - print "Swift tests disabled: OpenStack creds not found in the environment" + print("Swift tests disabled: OpenStack creds not found in the environment") swift_conn = None -except ImportError, e: - print "Swift tests disabled: unable to import swiftclient", e +except ImportError as e: + print("Swift tests disabled: unable to import swiftclient", e) swift_conn = None diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index 13fc379af..1e6ec41e7 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -18,6 +18,7 @@ class UpdateMirror1Test(BaseTest): """ update mirrors: regular update """ + sortOutput = True longTest = False fixtureCmds = [ "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish https://packagecloud.io/varnishcache/varnish30/debian/ wheezy main", @@ -25,9 +26,6 @@ class UpdateMirror1Test(BaseTest): runCmd = "aptly mirror update --ignore-signatures varnish" outputMatchPrepare = filterOutRedirects - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror2Test(BaseTest): """ @@ -113,6 +111,7 @@ class UpdateMirror7Test(BaseTest): """ update mirrors: flat repository """ + sortOutput = True fixtureGpg = True fixtureCmds = [ "aptly mirror create --keyring=aptlytest.gpg -architectures=amd64 flat https://cloud.r-project.org/bin/linux/debian jessie-cran35/", @@ -120,9 +119,6 @@ class UpdateMirror7Test(BaseTest): runCmd = "aptly mirror update --keyring=aptlytest.gpg flat" outputMatchPrepare = filterOutSignature - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror8Test(BaseTest): """ @@ -143,6 +139,7 @@ class UpdateMirror9Test(BaseTest): """ update mirrors: flat repository + sources """ + sortOutput = True fixtureGpg = True fixtureCmds = [ "aptly mirror create --keyring=aptlytest.gpg -with-sources flat-src https://cloud.r-project.org/bin/linux/debian jessie-cran35/", @@ -150,14 +147,12 @@ class UpdateMirror9Test(BaseTest): runCmd = "aptly mirror update --keyring=aptlytest.gpg flat-src" outputMatchPrepare = filterOutSignature - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror10Test(BaseTest): """ update mirrors: filtered """ + sortOutput = True fixtureGpg = True fixtureCmds = [ "aptly mirror create -keyring=aptlytest.gpg -with-sources -filter='!(Name (% r-*)), !($$PackageType (source))' flat-src https://cloud.r-project.org/bin/linux/debian jessie-cran35/", @@ -165,15 +160,13 @@ class UpdateMirror10Test(BaseTest): runCmd = "aptly mirror update --keyring=aptlytest.gpg flat-src" outputMatchPrepare = filterOutSignature - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror11Test(BaseTest): """ update mirrors: update over FTP """ skipTest = "Requesting obsolete file - stretch/InRelease" + sortOutput = True longTest = False fixtureGpg = True requiresFTP = True @@ -183,15 +176,13 @@ class UpdateMirror11Test(BaseTest): outputMatchPrepare = filterOutSignature runCmd = "aptly mirror update -keyring=aptlytest.gpg stretch-main" - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror12Test(BaseTest): """ update mirrors: update with udebs """ skipTest = "Requesting obsolete file - stretch/InRelease" + sortOutput = True longTest = False fixtureGpg = True fixtureCmds = [ @@ -200,14 +191,12 @@ class UpdateMirror12Test(BaseTest): runCmd = "aptly mirror update -keyring=aptlytest.gpg stretch" outputMatchPrepare = filterOutSignature - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror13Test(BaseTest): """ update mirrors: regular update with --skip-existing-packages option """ + sortOutput = True longTest = False fixtureCmds = [ "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish https://packagecloud.io/varnishcache/varnish30/debian/ wheezy main", @@ -215,14 +204,12 @@ class UpdateMirror13Test(BaseTest): runCmd = "aptly mirror update --ignore-signatures --skip-existing-packages varnish" outputMatchPrepare = filterOutRedirects - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror14Test(BaseTest): """ update mirrors: regular update with --skip-existing-packages option """ + sortOutput = True longTest = False fixtureCmds = [ "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish https://packagecloud.io/varnishcache/varnish30/debian/ wheezy main", @@ -231,24 +218,19 @@ class UpdateMirror14Test(BaseTest): runCmd = "aptly mirror update --ignore-signatures --skip-existing-packages varnish" outputMatchPrepare = filterOutRedirects - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror15Test(BaseTest): """ update mirrors: update for mirror without MD5 checksums """ skipTest = "Using deprecated service - bintray" + sortOutput = True longTest = False fixtureCmds = [ "aptly mirror create --ignore-signatures bintray https://dl.bintray.com/smira/deb/ ./", ] runCmd = "aptly mirror update --ignore-signatures bintray" - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - def check(self): super(UpdateMirror15Test, self).check() # check pool @@ -263,15 +245,13 @@ class UpdateMirror16Test(BaseTest): as mirror lacks MD5 checksum, file would be downloaded but not re-imported """ skipTest = "Using deprecated service - bintray" + sortOutput = True longTest = False fixtureCmds = [ "aptly mirror create --ignore-signatures bintray https://dl.bintray.com/smira/deb/ ./", ] runCmd = "aptly mirror update --ignore-signatures bintray" - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - def prepare(self): super(UpdateMirror16Test, self).prepare() @@ -292,15 +272,13 @@ class UpdateMirror17Test(BaseTest): """ update mirrors: update for mirror but with file in pool on legacy MD5 location """ + sortOutput = True longTest = False fixtureCmds = [ "aptly mirror create -ignore-signatures -architectures=i386 -filter=libboost-program-options-dev stretch http://cdn-fastly.deb.debian.org/debian stretch main", ] runCmd = "aptly mirror update -ignore-signatures stretch" - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - def prepare(self): super(UpdateMirror17Test, self).prepare() @@ -321,6 +299,7 @@ class UpdateMirror18Test(BaseTest): """ update mirrors: update for mirror but with file in pool on legacy MD5 location and disabled legacy path support """ + sortOutput = True longTest = False fixtureCmds = [ "aptly mirror create -ignore-signatures -architectures=i386 -filter=libboost-program-options-dev stretch http://cdn-fastly.deb.debian.org/debian stretch main", @@ -328,9 +307,6 @@ class UpdateMirror18Test(BaseTest): runCmd = "aptly mirror update -ignore-signatures stretch" configOverride = {'skipLegacyPool': True} - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - def prepare(self): super(UpdateMirror18Test, self).prepare() @@ -361,13 +337,14 @@ class UpdateMirror19Test(BaseTest): outputMatchPrepare = filterOutSignature def output_processor(self, output): - return "\n".join(line for line in output.split("\n") if ".deb" not in line) + return "\n".join(line for line in self.ensure_utf8(output).split("\n") if ".deb" not in line) class UpdateMirror20Test(BaseTest): """ update mirrors: flat repository (internal GPG implementation) """ + sortOutput = True fixtureGpg = True fixtureCmds = [ "aptly mirror create --keyring=aptlytest.gpg -architectures=amd64 --filter='r-cran-class' flat https://cloud.r-project.org/bin/linux/debian jessie-cran35/", @@ -376,9 +353,6 @@ class UpdateMirror20Test(BaseTest): runCmd = "aptly mirror update --keyring=aptlytest.gpg flat" outputMatchPrepare = filterOutSignature - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror21Test(BaseTest): """ @@ -395,7 +369,7 @@ class UpdateMirror21Test(BaseTest): outputMatchPrepare = filterOutSignature def output_processor(self, output): - return "\n".join(line for line in output.split("\n") if ".deb" not in line) + return "\n".join(line for line in self.ensure_utf8(output).split("\n") if ".deb" not in line) class UpdateMirror22Test(BaseTest): @@ -418,6 +392,7 @@ class UpdateMirror23Test(BaseTest): update mirrors: update with installer """ skipTest = "Requesting obsolete file - stretch/InRelease" + sortOutput = True longTest = False fixtureGpg = True fixtureCmds = [ @@ -426,15 +401,13 @@ class UpdateMirror23Test(BaseTest): runCmd = "aptly mirror update -keyring=aptlytest.gpg stretch" outputMatchPrepare = filterOutSignature - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class UpdateMirror24Test(BaseTest): """ update mirrors: update with installer with separate gpg file """ skipTest = "Requesting obsolete file - stretch/InRelease" + sortOutput = True longTest = False fixtureGpg = True fixtureCmds = [ @@ -442,6 +415,3 @@ class UpdateMirror24Test(BaseTest): ] runCmd = "aptly mirror update -keyring=aptlytest.gpg trusty" outputMatchPrepare = filterOutSignature - - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 57a765dd7..9b36f60e8 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -2,21 +2,13 @@ import hashlib import inspect import re -import zlib -from lib import BaseTest +from lib import BaseTest, ungzip_if_required def strip_processor(output): return "\n".join([l for l in output.split("\n") if not l.startswith(' ') and not l.startswith('Date:')]) -def ungzip_if_required(output): - if output.startswith("\x1f\x8b"): - return zlib.decompress(output, 16 + zlib.MAX_WBITS) - - return output - - class PublishRepo1Test(BaseTest): """ publish repo: default @@ -60,9 +52,9 @@ def check(self): self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n")))) self.check_file_contents('public/dists/maverick/main/Contents-i386.gz', - 'contents_i386', match_prepare=ungzip_if_required) + 'contents_i386', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False) self.check_file_contents('public/dists/maverick/Contents-i386.gz', - 'contents_i386_legacy', match_prepare=ungzip_if_required) + 'contents_i386_legacy', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False) # verify signatures self.run_cmd([self.gpgFinder.gpg, "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"), @@ -101,8 +93,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join( - 'public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % ( @@ -545,8 +536,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join( - 'public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % ( @@ -637,8 +627,8 @@ class PublishRepo23Test(BaseTest): runCmd = "aptly publish repo -component=main,contrib repo1" expectedCode = 2 - def outputMatchPrepare(_, s): - return "\n".join([l for l in s.split("\n") if l.startswith("ERROR")]) + def outputMatchPrepare(self, s): + return "\n".join([l for l in self.ensure_utf8(s).split("\n") if l.startswith("ERROR")]) class PublishRepo24Test(BaseTest): @@ -832,8 +822,8 @@ class PublishRepo31Test(BaseTest): gold_processor = BaseTest.expand_environ configOverride = {"gpgProvider": "internal"} - def outputMatchPrepare(_, s): - return re.sub(r' \d{4}-\d{2}-\d{2}', '', s) + def outputMatchPrepare(self, s): + return re.sub(r' \d{4}-\d{2}-\d{2}', '', self.ensure_utf8(s)) def check(self): super(PublishRepo31Test, self).check() diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index 0e4bdec93..69d1983d8 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -1,8 +1,7 @@ import os import hashlib import inspect -import zlib -from lib import BaseTest +from lib import BaseTest, ungzip_if_required def strip_processor(output): @@ -13,13 +12,6 @@ def sorted_processor(output): return "\n".join(sorted(output.split("\n"))) -def ungzip_if_required(output): - if output.startswith("\x1f\x8b"): - return zlib.decompress(output, 16 + zlib.MAX_WBITS) - - return output - - class PublishSnapshot1Test(BaseTest): """ publish snapshot: defaults @@ -75,9 +67,9 @@ def check(self): 'packages_amd64', match_prepare=sorted_processor) self.check_file_contents('public/dists/maverick/main/Contents-i386.gz', - 'contents_i386', match_prepare=ungzip_if_required) + 'contents_i386', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False) self.check_file_contents('public/dists/maverick/main/Contents-amd64.gz', - 'contents_amd64', match_prepare=ungzip_if_required) + 'contents_amd64', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False) # verify signatures self.run_cmd([self.gpgFinder.gpg, "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"), @@ -116,8 +108,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join( - 'public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % ( @@ -841,8 +832,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join( - 'public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % ( @@ -940,8 +930,8 @@ class PublishSnapshot32Test(BaseTest): runCmd = "aptly publish snapshot -component=main,contrib snap32.1" expectedCode = 2 - def outputMatchPrepare(_, s): - return "\n".join([l for l in s.split("\n") if l.startswith("ERROR")]) + def outputMatchPrepare(self, s): + return "\n".join([l for l in self.ensure_utf8(s).split("\n") if l.startswith("ERROR")]) class PublishSnapshot33Test(BaseTest): @@ -1093,7 +1083,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join('public/dists/stretch', path))) + h.update(self.read_file(os.path.join('public/dists/stretch', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % ( @@ -1217,6 +1207,6 @@ def check(self): 'packages_amd64', match_prepare=sorted_processor) self.check_file_contents('public/dists/maverick/main/Contents-i386.gz', - 'contents_i386', match_prepare=ungzip_if_required) + 'contents_i386', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False) self.check_file_contents('public/dists/maverick/main/Contents-amd64.gz', - 'contents_amd64', match_prepare=ungzip_if_required) + 'contents_amd64', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False) diff --git a/system/t06_publish/switch.py b/system/t06_publish/switch.py index d5178c487..4f8733977 100644 --- a/system/t06_publish/switch.py +++ b/system/t06_publish/switch.py @@ -82,7 +82,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join('public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) @@ -346,7 +346,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join('public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) @@ -378,8 +378,8 @@ class PublishSwitch9Test(BaseTest): runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=a,b maverick snap2" expectedCode = 2 - def outputMatchPrepare(_, s): - return "\n".join([l for l in s.split("\n") if l.startswith("ERROR")]) + def outputMatchPrepare(self, s): + return "\n".join([l for l in self.ensure_utf8(s).split("\n") if l.startswith("ERROR")]) class PublishSwitch10Test(BaseTest): @@ -535,7 +535,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join('public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) diff --git a/system/t06_publish/update.py b/system/t06_publish/update.py index 752f74bcc..b2acbfe86 100644 --- a/system/t06_publish/update.py +++ b/system/t06_publish/update.py @@ -82,7 +82,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join('public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) @@ -427,7 +427,7 @@ def check(self): else: h = hashlib.sha512() - h.update(self.read_file(os.path.join('public/dists/maverick', path))) + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) if h.hexdigest() != fileHash: raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) diff --git a/system/t07_serve/__init__.py b/system/t07_serve/__init__.py index 9d2bfdcd6..457b9f593 100644 --- a/system/t07_serve/__init__.py +++ b/system/t07_serve/__init__.py @@ -2,7 +2,7 @@ Testing serving public repo """ -import httplib +import http.client import os import signal import subprocess @@ -50,7 +50,7 @@ def run(self): try: time.sleep(1) - conn = httplib.HTTPConnection("127.0.0.1", 8765) + conn = http.client.HTTPConnection("127.0.0.1", 8765) conn.request("GET", "/") r = conn.getresponse() if r.status != 200: @@ -66,7 +66,7 @@ def run(self): if proc.returncode != -2 and proc.returncode != 2: raise Exception("exit code %d != %d (output: %s)" % (proc.returncode, -2, output)) self.output = output - except Exception, e: + except Exception as e: raise Exception("Running command %s failed: %s" % (self.runCmd, str(e))) def check(self): diff --git a/system/t09_repo/add.py b/system/t09_repo/add.py index dea199b9c..61c64d1ea 100644 --- a/system/t09_repo/add.py +++ b/system/t09_repo/add.py @@ -76,8 +76,8 @@ def prepare(self): super(AddRepo4Test, self).prepare() self.tempSrcDir = tempfile.mkdtemp() - os.makedirs(os.path.join(self.tempSrcDir, "01"), 0755) - os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0755) + os.makedirs(os.path.join(self.tempSrcDir, "01"), 0o755) + os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0o755) shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "libboost-program-options-dev_1.49.0.1_i386.deb"), os.path.join(self.tempSrcDir, "01")) @@ -133,7 +133,7 @@ def prepare(self): super(AddRepo5Test, self).prepare() self.tempSrcDir = tempfile.mkdtemp() - os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0755) + os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0o755) shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1-1.3.dsc"), os.path.join(self.tempSrcDir, "02", "03")) @@ -330,7 +330,7 @@ def prepare(self): super(AddRepo16Test, self).prepare() self.tempSrcDir = tempfile.mkdtemp() - os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0755) + os.makedirs(os.path.join(self.tempSrcDir, "02", "03"), 0o755) shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pyspi_0.6.1-1.3.dsc"), os.path.join(self.tempSrcDir, "02", "03")) diff --git a/system/t09_repo/cmdimport.py b/system/t09_repo/cmdimport.py index d879d0e41..5a4337baf 100644 --- a/system/t09_repo/cmdimport.py +++ b/system/t09_repo/cmdimport.py @@ -5,6 +5,7 @@ class ImportRepo1Test(BaseTest): """ import to local repo: simple import """ + sortOutput = True fixtureDB = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", @@ -16,14 +17,12 @@ def check(self): self.check_output() self.check_cmd_output("aptly repo show -with-packages repo1", "repo_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class ImportRepo2Test(BaseTest): """ import to local repo: import w/deps """ + sortOutput = True fixtureDB = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", @@ -35,14 +34,12 @@ def check(self): self.check_output() self.check_cmd_output("aptly repo show -with-packages repo1", "repo_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class ImportRepo3Test(BaseTest): """ import to local repo: simple move w/deps but w/o archs """ + sortOutput = True fixtureDB = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", @@ -50,14 +47,12 @@ class ImportRepo3Test(BaseTest): runCmd = "aptly repo import -with-deps wheezy-contrib repo1 redeclipse" expectedCode = 1 - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class ImportRepo4Test(BaseTest): """ import to local repo: dry run """ + sortOutput = True fixtureDB = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", @@ -68,14 +63,12 @@ def check(self): self.check_output() self.check_cmd_output("aptly repo show -with-packages repo1", "repo_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class ImportRepo5Test(BaseTest): """ import to local repo: wrong dep """ + sortOutput = True fixtureDB = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", @@ -83,9 +76,6 @@ class ImportRepo5Test(BaseTest): runCmd = "aptly repo import wheezy-contrib repo1 'pyspi >> 0.6.1-1.3)'" expectedCode = 1 - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class ImportRepo6Test(BaseTest): """ @@ -125,11 +115,9 @@ class ImportRepo9Test(BaseTest): """ import to local repo: import with complex query """ + sortOutput = True fixtureDB = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", ] runCmd = "aptly repo import wheezy-main repo1 '(httpd, $$Source (=nginx)) | exim4'" - - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) diff --git a/system/t09_repo/copy.py b/system/t09_repo/copy.py index 9280f61c7..40e5c7dd5 100644 --- a/system/t09_repo/copy.py +++ b/system/t09_repo/copy.py @@ -5,6 +5,7 @@ class CopyRepo1Test(BaseTest): """ copy in local repo: simple copy """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -17,14 +18,12 @@ def check(self): self.check_cmd_output("aptly repo show -with-packages repo1", "repo1_show") self.check_cmd_output("aptly repo show -with-packages repo2", "repo2_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class CopyRepo2Test(BaseTest): """ copy in local repo: simple copy w/deps """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -37,14 +36,12 @@ def check(self): self.check_cmd_output("aptly repo show -with-packages repo1", "repo1_show") self.check_cmd_output("aptly repo show -with-packages repo2", "repo2_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class CopyRepo3Test(BaseTest): """ copy in local repo: simple copy w/deps but w/o archs """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -53,14 +50,12 @@ class CopyRepo3Test(BaseTest): runCmd = "aptly repo copy -with-deps repo1 repo2 'pyspi (>> 0.6.1-1.3)' libboost-program-options-dev_1.49.0.1_i386" expectedCode = 1 - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class CopyRepo4Test(BaseTest): """ copy in local repo: dry run """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -73,14 +68,12 @@ def check(self): self.check_cmd_output("aptly repo show -with-packages repo1", "repo1_show") self.check_cmd_output("aptly repo show -with-packages repo2", "repo2_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class CopyRepo5Test(BaseTest): """ copy in local repo: wrong dep """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -89,9 +82,6 @@ class CopyRepo5Test(BaseTest): runCmd = "aptly repo copy repo1 repo2 'pyspi >> 0.6.1-1.3)'" expectedCode = 1 - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class CopyRepo6Test(BaseTest): """ diff --git a/system/t09_repo/include.py b/system/t09_repo/include.py index 2ae6b99a8..0abe383ca 100644 --- a/system/t09_repo/include.py +++ b/system/t09_repo/include.py @@ -145,7 +145,7 @@ def prepare(self): super(IncludeRepo6Test, self).prepare() self.tempSrcDir = tempfile.mkdtemp() - os.makedirs(os.path.join(self.tempSrcDir, "01"), 0755) + os.makedirs(os.path.join(self.tempSrcDir, "01"), 0o755) for path in ["hardlink_0.2.1.dsc", "hardlink_0.2.1_amd64.changes", "hardlink_0.2.1_amd64.deb"]: shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path), @@ -536,7 +536,7 @@ def prepare(self): super(IncludeRepo22Test, self).prepare() self.tempSrcDir = tempfile.mkdtemp() - os.makedirs(os.path.join(self.tempSrcDir, "01"), 0755) + os.makedirs(os.path.join(self.tempSrcDir, "01"), 0o755) for path in ["hardlink_0.2.1.dsc", "hardlink_0.2.1_amd64.deb"]: shutil.copy(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes", path), diff --git a/system/t09_repo/move.py b/system/t09_repo/move.py index 301369dcf..9d9957d99 100644 --- a/system/t09_repo/move.py +++ b/system/t09_repo/move.py @@ -5,6 +5,7 @@ class MoveRepo1Test(BaseTest): """ move in local repo: simple move """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -17,14 +18,12 @@ def check(self): self.check_cmd_output("aptly repo show -with-packages repo1", "repo1_show") self.check_cmd_output("aptly repo show -with-packages repo2", "repo2_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class MoveRepo2Test(BaseTest): """ move in local repo: simple move w/deps """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -37,14 +36,12 @@ def check(self): self.check_cmd_output("aptly repo show -with-packages repo1", "repo1_show") self.check_cmd_output("aptly repo show -with-packages repo2", "repo2_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class MoveRepo3Test(BaseTest): """ move in local repo: simple move w/deps but w/o archs """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -53,14 +50,12 @@ class MoveRepo3Test(BaseTest): runCmd = "aptly repo move -with-deps repo1 repo2 'pyspi (>> 0.6.1-1.3)' libboost-program-options-dev_1.49.0.1_i386" expectedCode = 1 - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class MoveRepo4Test(BaseTest): """ move in local repo: dry run """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -73,14 +68,12 @@ def check(self): self.check_cmd_output("aptly repo show -with-packages repo1", "repo1_show") self.check_cmd_output("aptly repo show -with-packages repo2", "repo2_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class MoveRepo5Test(BaseTest): """ move in local repo: wrong dep """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze repo1", "aptly repo create -comment=Cool -distribution=squeeze repo2", @@ -89,9 +82,6 @@ class MoveRepo5Test(BaseTest): runCmd = "aptly repo move repo1 repo2 'pyspi >> 0.6.1-1.3)'" expectedCode = 1 - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class MoveRepo6Test(BaseTest): """ diff --git a/system/t09_repo/remove.py b/system/t09_repo/remove.py index 5acb1d79d..7bbf570a0 100644 --- a/system/t09_repo/remove.py +++ b/system/t09_repo/remove.py @@ -5,6 +5,7 @@ class RemoveRepo1Test(BaseTest): """ remove from local repo: as dep """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze local-repo", "aptly repo add local-repo ${files}" @@ -15,14 +16,12 @@ def check(self): self.check_output() self.check_cmd_output("aptly repo show -with-packages local-repo", "repo_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class RemoveRepo2Test(BaseTest): """ remove from local repo: as dep with version, key """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze local-repo", "aptly repo add local-repo ${files}" @@ -33,9 +32,6 @@ def check(self): self.check_output() self.check_cmd_output("aptly repo show -with-packages local-repo", "repo_show") - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) - class RemoveRepo3Test(BaseTest): """ @@ -49,6 +45,7 @@ class RemoveRepo4Test(BaseTest): """ remove from local repo: dry run """ + sortOutput = True fixtureCmds = [ "aptly repo create -comment=Cool -distribution=squeeze local-repo", "aptly repo add local-repo ${files}" @@ -58,6 +55,3 @@ class RemoveRepo4Test(BaseTest): def check(self): self.check_output() self.check_cmd_output("aptly repo show -with-packages local-repo", "repo_show") - - def output_processor(self, output): - return "\n".join(sorted(output.split("\n"))) diff --git a/system/t12_api/mirrors.py b/system/t12_api/mirrors.py index 869ec8cd3..33025462a 100644 --- a/system/t12_api/mirrors.py +++ b/system/t12_api/mirrors.py @@ -10,11 +10,11 @@ class MirrorsAPITestCreateShow(APITest): def check(self): mirror_name = self.random_name() - mirror_desc = {u'Name': mirror_name, - u'ArchiveURL': 'http://security.debian.org/', - u'Architectures': ['amd64'], - u'Components': ['main'], - u'Distribution': 'wheezy/updates'} + mirror_desc = {'Name': mirror_name, + 'ArchiveURL': 'http://security.debian.org/', + 'Architectures': ['amd64'], + 'Components': ['main'], + 'Distribution': 'wheezy/updates'} resp = self.post("/api/mirrors", json=mirror_desc) self.check_equal(resp.status_code, 400) @@ -22,17 +22,17 @@ def check(self): 'error': 'unable to fetch mirror: verification of detached signature failed: exit status 2', }, resp.json()) - mirror_desc[u'IgnoreSignatures'] = True + mirror_desc['IgnoreSignatures'] = True resp = self.post("/api/mirrors", json=mirror_desc) self.check_equal(resp.status_code, 201) resp = self.get("/api/mirrors/" + mirror_name) self.check_equal(resp.status_code, 200) - self.check_subset({u'Name': mirror_name, - u'ArchiveRoot': 'http://security.debian.org/', - u'Architectures': ['amd64'], - u'Components': ['main'], - u'Distribution': 'wheezy/updates'}, resp.json()) + self.check_subset({'Name': mirror_name, + 'ArchiveRoot': 'http://security.debian.org/', + 'Architectures': ['amd64'], + 'Components': ['main'], + 'Distribution': 'wheezy/updates'}, resp.json()) resp = self.get("/api/mirrors/" + mirror_desc["Name"] + "/packages") self.check_equal(resp.status_code, 404) @@ -44,12 +44,12 @@ class MirrorsAPITestCreateUpdate(APITest): """ def check(self): mirror_name = self.random_name() - mirror_desc = {u'Name': mirror_name, - u'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', - u'Distribution': 'wheezy', - u'Components': ['main']} + mirror_desc = {'Name': mirror_name, + 'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', + 'Distribution': 'wheezy', + 'Components': ['main']} - mirror_desc[u'IgnoreSignatures'] = True + mirror_desc['IgnoreSignatures'] = True resp = self.post("/api/mirrors", json=mirror_desc) self.check_equal(resp.status_code, 201) @@ -68,9 +68,9 @@ def check(self): resp = self.get("/api/mirrors/" + mirror_desc["Name"]) self.check_equal(resp.status_code, 200) - self.check_subset({u'Name': mirror_desc["Name"], - u'ArchiveRoot': 'https://packagecloud.io/varnishcache/varnish30/debian/', - u'Distribution': 'wheezy'}, resp.json()) + self.check_subset({'Name': mirror_desc["Name"], + 'ArchiveRoot': 'https://packagecloud.io/varnishcache/varnish30/debian/', + 'Distribution': 'wheezy'}, resp.json()) resp = self.get("/api/mirrors/" + mirror_desc["Name"] + "/packages") self.check_equal(resp.status_code, 200) @@ -82,11 +82,11 @@ class MirrorsAPITestCreateDelete(APITest): """ def check(self): mirror_name = self.random_name() - mirror_desc = {u'Name': mirror_name, - u'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', - u'IgnoreSignatures': True, - u'Distribution': 'wheezy', - u'Components': ['main']} + mirror_desc = {'Name': mirror_name, + 'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', + 'IgnoreSignatures': True, + 'Distribution': 'wheezy', + 'Components': ['main']} resp = self.post("/api/mirrors", json=mirror_desc) self.check_equal(resp.status_code, 201) @@ -105,11 +105,11 @@ def check(self): count = len(resp.json()) mirror_name = self.random_name() - mirror_desc = {u'Name': mirror_name, - u'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', - u'IgnoreSignatures': True, - u'Distribution': 'wheezy', - u'Components': ['main']} + mirror_desc = {'Name': mirror_name, + 'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', + 'IgnoreSignatures': True, + 'Distribution': 'wheezy', + 'Components': ['main']} resp = self.post("/api/mirrors", json=mirror_desc) self.check_equal(resp.status_code, 201) diff --git a/system/t12_api/packages.py b/system/t12_api/packages.py index 836d939d2..fea119517 100644 --- a/system/t12_api/packages.py +++ b/system/t12_api/packages.py @@ -1,4 +1,7 @@ -import urllib +import urllib.error +import urllib.parse +import urllib.request + from api_lib import APITest @@ -19,7 +22,7 @@ def check(self): self.check_equal(resp.json()['State'], 2) # get information about package - resp = self.get("/api/packages/" + urllib.quote('Psource pyspi 0.6.1-1.3 3a8b37cbd9a3559e')) + resp = self.get("/api/packages/" + urllib.parse.quote('Psource pyspi 0.6.1-1.3 3a8b37cbd9a3559e')) self.check_equal(resp.status_code, 200) self.check_equal(resp.json(), { 'Architecture': 'any', @@ -40,5 +43,5 @@ def check(self): 'Vcs-Svn': 'svn://svn.tribulaciones.org/srv/svn/pyspi/trunk', 'Version': '0.6.1-1.3'}) - resp = self.get("/api/packages/" + urllib.quote('Pamd64 no-such-package 1.0 3a8b37cbd9a3559e')) + resp = self.get("/api/packages/" + urllib.parse.quote('Pamd64 no-such-package 1.0 3a8b37cbd9a3559e')) self.check_equal(resp.status_code, 404) diff --git a/system/t12_api/repos.py b/system/t12_api/repos.py index 8851c71d7..46ab03ddb 100644 --- a/system/t12_api/repos.py +++ b/system/t12_api/repos.py @@ -1,5 +1,5 @@ from api_lib import APITest -from publish import DefaultSigningOptions +from .publish import DefaultSigningOptions class ReposAPITestCreateShow(APITest): @@ -8,10 +8,10 @@ class ReposAPITestCreateShow(APITest): """ def check(self): repo_name = self.random_name() - repo_desc = {u'Comment': u'fun repo', - u'DefaultComponent': u'', - u'DefaultDistribution': u'', - u'Name': repo_name} + repo_desc = {'Comment': 'fun repo', + 'DefaultComponent': '', + 'DefaultDistribution': '', + 'Name': repo_name} resp = self.post("/api/repos", json={"Name": repo_name, "Comment": "fun repo"}) self.check_equal(resp.json(), repo_desc) @@ -101,10 +101,10 @@ def check(self): resp = self.get("/api/tasks/" + str(resp.json()['ID']) + "/output") self.check_equal(resp.status_code, 200) - self.check_in("Added: pyspi_0.6.1-1.3_source added", resp.content) - self.check_equal("Removed: " in resp.content, False) - self.check_equal("Failed files: " in resp.content, False) - self.check_equal("Warnings: " in resp.content, False) + self.check_in(b"Added: pyspi_0.6.1-1.3_source added", resp.content) + self.check_not_in(b"Removed: ", resp.content) + self.check_not_in(b"Failed files: ", resp.content) + self.check_not_in(b"Warnings: ", resp.content) self.check_equal(self.get("/api/repos/" + repo_name + "/packages").json(), ['Psource pyspi 0.6.1-1.3 3a8b37cbd9a3559e']) @@ -169,10 +169,10 @@ def check(self): resp = self.get("/api/tasks/" + str(resp.json()['ID']) + "/output") self.check_equal(resp.status_code, 200) - self.check_in("Added: libboost-program-options-dev_1.49.0.1_i386 added", resp.content) - self.check_equal("Removed: " in resp.content, False) - self.check_equal("Failed files: " in resp.content, False) - self.check_equal("Warnings: " in resp.content, False) + self.check_in(b"Added: libboost-program-options-dev_1.49.0.1_i386 added", resp.content) + self.check_not_in(b"Removed: ", resp.content) + self.check_not_in(b"Failed files: ", resp.content) + self.check_not_in(b"Warnings: ", resp.content) self.check_equal(self.get("/api/repos/" + repo_name + "/packages").json(), ['Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378']) @@ -200,10 +200,10 @@ def check(self): resp = self.get("/api/tasks/" + str(resp.json()['ID']) + "/output") self.check_equal(resp.status_code, 200) - self.check_in("Added: hardlink_0.2.1_source added, hardlink_0.2.1_amd64 added", resp.content) + self.check_in(b"Added: hardlink_0.2.1_source added, hardlink_0.2.1_amd64 added", resp.content) self.check_equal( sorted(self.get("/api/repos/" + repo_name + "/packages").json()), - [u'Pamd64 hardlink 0.2.1 daf8fcecbf8210ad', u'Psource hardlink 0.2.1 8f72df429d7166e5'] + ['Pamd64 hardlink 0.2.1 daf8fcecbf8210ad', 'Psource hardlink 0.2.1 8f72df429d7166e5'] ) self.check_not_exists("upload/" + d) @@ -238,7 +238,7 @@ def check(self): resp = self.get("/api/repos/" + repo_name + "/packages", params={"q": "pyspi)"}) self.check_equal(resp.status_code, 400) - self.check_equal(resp.json()["error"], u'parsing failed: unexpected token ): expecting end of query') + self.check_equal(resp.json()["error"], 'parsing failed: unexpected token ): expecting end of query') class ReposAPITestAddMultiple(APITest): diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index 16b0f3775..17e67bcf4 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -1,5 +1,5 @@ from api_lib import APITest -from publish import DefaultSigningOptions +from .publish import DefaultSigningOptions class SnapshotsAPITestCreateShowEmpty(APITest): @@ -8,8 +8,8 @@ class SnapshotsAPITestCreateShowEmpty(APITest): """ def check(self): snapshot_name = self.random_name() - snapshot_desc = {u'Description': u'fun snapshot', - u'Name': snapshot_name} + snapshot_desc = {'Description': 'fun snapshot', + 'Name': snapshot_name} # create empty snapshot resp = self.post_task("/api/snapshots", json=snapshot_desc) @@ -36,9 +36,9 @@ class SnapshotsAPITestCreateFromRefs(APITest): """ def check(self): snapshot_name = self.random_name() - snapshot_desc = {u'Description': u'fun snapshot', - u'Name': snapshot_name, - u'SourceSnapshots': [self.random_name()]} + snapshot_desc = {'Description': 'fun snapshot', + 'Name': snapshot_name, + 'SourceSnapshots': [self.random_name()]} # creating snapshot from missing source snapshot resp = self.post("/api/snapshots", json=snapshot_desc) @@ -115,15 +115,15 @@ def check(self): self.check_equal(resp.json()['State'], 2) self.check_equal(self.get("/api/snapshots/" + snapshot_name).status_code, 200) - self.check_subset({u'Architecture': 'i386', - u'Package': 'libboost-program-options-dev', - u'Version': '1.49.0.1', + self.check_subset({'Architecture': 'i386', + 'Package': 'libboost-program-options-dev', + 'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, self.get("/api/snapshots/" + snapshot_name + "/packages", params={"format": "details"}).json()[0]) - self.check_subset({u'Architecture': 'i386', - u'Package': 'libboost-program-options-dev', - u'Version': '1.49.0.1', + self.check_subset({'Architecture': 'i386', + 'Package': 'libboost-program-options-dev', + 'Version': '1.49.0.1', 'FilesHash': '918d2f433384e378'}, self.get("/api/snapshots/" + snapshot_name + "/packages", params={"format": "details", "q": "Version (> 0.6.1-1.4)"}).json()[0]) @@ -139,8 +139,8 @@ class SnapshotsAPITestCreateUpdate(APITest): """ def check(self): snapshot_name = self.random_name() - snapshot_desc = {u'Description': u'fun snapshot', - u'Name': snapshot_name} + snapshot_desc = {'Description': 'fun snapshot', + 'Name': snapshot_name} resp = self.post_task("/api/snapshots", json=snapshot_desc) self.check_equal(resp.json()['State'], 2) @@ -171,8 +171,8 @@ class SnapshotsAPITestCreateDelete(APITest): """ def check(self): snapshot_name = self.random_name() - snapshot_desc = {u'Description': u'fun snapshot', - u'Name': snapshot_name} + snapshot_desc = {'Description': 'fun snapshot', + 'Name': snapshot_name} # deleting unreferenced snapshot resp = self.post_task("/api/snapshots", json=snapshot_desc) @@ -251,8 +251,8 @@ class SnapshotsAPITestDiff(APITest): GET /api/snapshot/:name/diff/:name2 """ def check(self): - repos = [self.random_name() for x in xrange(2)] - snapshots = [self.random_name() for x in xrange(2)] + repos = [self.random_name() for x in range(2)] + snapshots = [self.random_name() for x in range(2)] for repo_name in repos: self.check_equal(self.post("/api/repos", json={"Name": repo_name}).status_code, 201) diff --git a/system/t12_api/systemd_handover.py b/system/t12_api/systemd_handover.py index 3d6de693f..eb3d0aac5 100644 --- a/system/t12_api/systemd_handover.py +++ b/system/t12_api/systemd_handover.py @@ -1,5 +1,7 @@ import requests_unixsocket -import urllib +import urllib.error +import urllib.parse +import urllib.request import os import os.path @@ -42,5 +44,5 @@ def check(self): print("Skipping test as we failed to setup a listener.") return session = requests_unixsocket.Session() - r = session.get('http+unix://%s/api/version' % urllib.quote(self.socket_path, safe='')) + r = session.get('http+unix://%s/api/version' % urllib.parse.quote(self.socket_path, safe='')) self.check_equal(r.json(), {'Version': os.environ['APTLY_VERSION']}) diff --git a/system/t12_api/tasks.py b/system/t12_api/tasks.py index 81c9b9852..5f184c078 100644 --- a/system/t12_api/tasks.py +++ b/system/t12_api/tasks.py @@ -1,5 +1,5 @@ from api_lib import APITest -from publish import DefaultSigningOptions +from .publish import DefaultSigningOptions class TaskAPITestParallelTasks(APITest): @@ -8,11 +8,11 @@ class TaskAPITestParallelTasks(APITest): """ def _create_mirror(self, dist): mirror_name = self.random_name() - mirror_desc = {u'Name': mirror_name, - u'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', - u'Distribution': dist, - u'Components': ['main']} - mirror_desc[u'IgnoreSignatures'] = True + mirror_desc = {'Name': mirror_name, + 'ArchiveURL': 'https://packagecloud.io/varnishcache/varnish30/debian/', + 'Distribution': dist, + 'Components': ['main']} + mirror_desc['IgnoreSignatures'] = True resp = self.post("/api/mirrors", json=mirror_desc) self.check_equal(resp.status_code, 201) resp = self.put("/api/mirrors/" + mirror_name, json=mirror_desc, params={'_async': True}) diff --git a/system/t12_api/unix_socket.py b/system/t12_api/unix_socket.py index 9a3f1372c..b14fe9465 100644 --- a/system/t12_api/unix_socket.py +++ b/system/t12_api/unix_socket.py @@ -1,7 +1,9 @@ import requests_unixsocket import time import os -import urllib +import urllib.error +import urllib.parse +import urllib.request from lib import BaseTest @@ -32,7 +34,7 @@ def run(self): """ def check(self): session = requests_unixsocket.Session() - r = session.get('http+unix://%s/api/version' % urllib.quote(UnixSocketAPITest.socket_path, safe='')) + r = session.get('http+unix://%s/api/version' % urllib.parse.quote(UnixSocketAPITest.socket_path, safe='')) # Just needs to come back, we actually don't care much about the code. # Only needs to verify that the socket is actually responding. self.check_equal(r.json(), {'Version': os.environ['APTLY_VERSION']})