From c50b11b3caaf32c9be699f24956628feab3ce9b2 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 18 Sep 2023 16:56:29 +0300 Subject: [PATCH 1/8] Attempt download from pillow-depends mirror first --- winbuild/build_prepare.py | 44 ++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index d9122e680a2..c15d0e5831c 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -421,25 +421,41 @@ def find_msvs(): } -def extract_dep(url, filename): - import tarfile +def download_dep(url: str, file: str) -> None: import urllib.request + + ex = None + for i in range(3): + try: + print(f"Fetching {url} (attempt {i + 1})...") + content = urllib.request.urlopen(url).read() + with open(file, "wb") as f: + f.write(content) + break + except urllib.error.URLError as e: + ex = e + + if ex: + raise RuntimeError(ex) + + +def extract_dep(url: str, filename: str) -> None: + import tarfile import zipfile file = os.path.join(args.depends_dir, filename) if not os.path.exists(file): - ex = None - for i in range(3): - try: - print("Fetching %s (attempt %d)..." % (url, i + 1)) - content = urllib.request.urlopen(url).read() - with open(file, "wb") as f: - f.write(content) - break - except urllib.error.URLError as e: - ex = e - else: - raise RuntimeError(ex) + # First try our mirror + mirror_url = ( + f"https://raw.githubusercontent.com/" + f"python-pillow/pillow-depends/main/{filename}" + ) + try: + download_dep(mirror_url, file) + except RuntimeError as exc: + # Otherwise try upstream + print(exc) + download_dep(url, file) print("Extracting " + filename) sources_dir_abs = os.path.abspath(sources_dir) From 66bf71bafaccb24b47bc1100216fa5cbaf716b99 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 18 Sep 2023 17:03:18 +0300 Subject: [PATCH 2/8] Add type hints --- winbuild/build_prepare.py | 43 +++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index c15d0e5831c..d6040d40879 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import os import platform @@ -7,36 +9,40 @@ import subprocess -def cmd_cd(path): +def cmd_cd(path: str) -> str: return f"cd /D {path}" -def cmd_set(name, value): +def cmd_set(name: str, value: str) -> str: return f"set {name}={value}" -def cmd_append(name, value): +def cmd_append(name: str, value: str) -> str: op = "path " if name == "PATH" else f"set {name}=" return op + f"%{name}%;{value}" -def cmd_copy(src, tgt): +def cmd_copy(src: str, tgt: str) -> str: return f'copy /Y /B "{src}" "{tgt}"' -def cmd_xcopy(src, tgt): +def cmd_xcopy(src: str, tgt: str) -> str: return f'xcopy /Y /E "{src}" "{tgt}"' -def cmd_mkdir(path): +def cmd_mkdir(path: str) -> str: return f'mkdir "{path}"' -def cmd_rmdir(path): +def cmd_rmdir(path: str) -> str: return f'rmdir /S /Q "{path}"' -def cmd_nmake(makefile=None, target="", params=None): +def cmd_nmake( + makefile: str | None = None, + target: str = "", + params: str | list[str] | tuple[str, ...] = None, +) -> str: if params is None: params = "" elif isinstance(params, (list, tuple)): @@ -55,7 +61,7 @@ def cmd_nmake(makefile=None, target="", params=None): ) -def cmds_cmake(target, *params): +def cmds_cmake(target: str | tuple[str, ...] | list[str], *params) -> list[str]: if not isinstance(target, str): target = " ".join(target) @@ -80,8 +86,11 @@ def cmds_cmake(target, *params): def cmd_msbuild( - file, configuration="Release", target="Build", platform="{msbuild_arch}" -): + file: str, + configuration: str = "Release", + target: str = "Build", + platform: str = "{msbuild_arch}", +) -> str: return " ".join( [ "{msbuild}", @@ -365,7 +374,7 @@ def cmd_msbuild( # based on distutils._msvccompiler from CPython 3.7.4 -def find_msvs(): +def find_msvs() -> dict[str, str] | None: root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") if not root: print("Program Files not found") @@ -482,7 +491,7 @@ def extract_dep(url: str, filename: str) -> None: raise RuntimeError(msg) -def write_script(name, lines): +def write_script(name: str, lines: list[str]) -> None: name = os.path.join(args.build_dir, name) lines = [line.format(**prefs) for line in lines] print("Writing " + name) @@ -493,7 +502,7 @@ def write_script(name, lines): print(" " + line) -def get_footer(dep): +def get_footer(dep: dict) -> list[str]: lines = [] for out in dep.get("headers", []): lines.append(cmd_copy(out, "{inc_dir}")) @@ -504,7 +513,7 @@ def get_footer(dep): return lines -def build_env(): +def build_env() -> None: lines = [ "if defined DISTUTILS_USE_SDK goto end", cmd_set("INCLUDE", "{inc_dir}"), @@ -520,7 +529,7 @@ def build_env(): write_script("build_env.cmd", lines) -def build_dep(name): +def build_dep(name: str) -> str: dep = deps[name] dir = dep["dir"] file = f"build_dep_{name}.cmd" @@ -570,7 +579,7 @@ def build_dep(name): return file -def build_dep_all(): +def build_dep_all() -> None: lines = [r'call "{build_dir}\build_env.cmd"'] for dep_name in deps: print() From cf79e2c3d3f2c3ef743776707cd79ec36657d946 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 18 Sep 2023 17:06:30 +0300 Subject: [PATCH 3/8] Capitalise constants --- winbuild/build_prepare.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index d6040d40879..34bfaa964f3 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -105,14 +105,14 @@ def cmd_msbuild( SF_PROJECTS = "https://sourceforge.net/projects" -architectures = { +ARCHITECTURES = { "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, "ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"}, } # dependencies, listed in order of compilation -deps = { +DEPS = { "libjpeg": { "url": SF_PROJECTS + "/libjpeg-turbo/files/3.0.0/libjpeg-turbo-3.0.0.tar.gz/download", @@ -530,7 +530,7 @@ def build_env() -> None: def build_dep(name: str) -> str: - dep = deps[name] + dep = DEPS[name] dir = dep["dir"] file = f"build_dep_{name}.cmd" @@ -581,7 +581,7 @@ def build_dep(name: str) -> str: def build_dep_all() -> None: lines = [r'call "{build_dir}\build_env.cmd"'] - for dep_name in deps: + for dep_name in DEPS: print() if dep_name in disabled: print(f"Skipping disabled dependency {dep_name}") @@ -627,7 +627,7 @@ def build_dep_all() -> None: ) parser.add_argument( "--architecture", - choices=architectures, + choices=ARCHITECTURES, default=os.environ.get( "ARCHITECTURE", ( @@ -659,7 +659,7 @@ def build_dep_all() -> None: ) args = parser.parse_args() - arch_prefs = architectures[args.architecture] + arch_prefs = ARCHITECTURES[args.architecture] print("Target architecture:", args.architecture) msvs = find_msvs() @@ -718,7 +718,7 @@ def build_dep_all() -> None: # TODO find NASM automatically } - for k, v in deps.items(): + for k, v in DEPS.items(): prefs[f"dir_{k}"] = os.path.join(sources_dir, v["dir"]) print() From 82c3999bc969225d616b0cea370d8d6e27ab7896 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 18 Sep 2023 17:10:29 +0300 Subject: [PATCH 4/8] Don't download entire pillow-depends.zip (851 MB) --- .appveyor.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 60132a9a35a..cc4d56d0bc9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -21,13 +21,11 @@ environment: install: - '%PYTHON%\%EXECUTABLE% --version' - '%PYTHON%\%EXECUTABLE% -m pip install --upgrade pip' -- curl -fsSL -o pillow-depends.zip https://github.com/python-pillow/pillow-depends/archive/main.zip - curl -fsSL -o pillow-test-images.zip https://github.com/python-pillow/test-images/archive/main.zip -- 7z x pillow-depends.zip -oc:\ - 7z x pillow-test-images.zip -oc:\ -- mv c:\pillow-depends-main c:\pillow-depends - xcopy /S /Y c:\test-images-main\* c:\pillow\tests\images -- 7z x ..\pillow-depends\nasm-2.16.01-win64.zip -oc:\ +- curl -fsSL -o nasm-win64.zip https://raw.githubusercontent.com/python-pillow/pillow-depends/main/nasm-2.16.01-win64.zip +- 7z x nasm-win64.zip -oc:\ - choco install ghostscript --version=10.0.0.20230317 - path c:\nasm-2.16.01;C:\Program Files\gs\gs10.00.0\bin;%PATH% - cd c:\pillow\winbuild\ From 15b1d6085c87d19f6f76c2c94b00d8940be22dd0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 18 Sep 2023 18:17:47 +0300 Subject: [PATCH 5/8] Use curl with --ssl-no-revoke To fix: curl: (35) schannel: next InitializeSecurityContext failed: Unknown error (0x80092013) - The revocation function was unable to check revocation because the revocation server was offline. --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index cc4d56d0bc9..4546e2a5bb0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -21,10 +21,10 @@ environment: install: - '%PYTHON%\%EXECUTABLE% --version' - '%PYTHON%\%EXECUTABLE% -m pip install --upgrade pip' -- curl -fsSL -o pillow-test-images.zip https://github.com/python-pillow/test-images/archive/main.zip +- curl -fsSL --ssl-no-revoke -o pillow-test-images.zip https://github.com/python-pillow/test-images/archive/main.zip - 7z x pillow-test-images.zip -oc:\ - xcopy /S /Y c:\test-images-main\* c:\pillow\tests\images -- curl -fsSL -o nasm-win64.zip https://raw.githubusercontent.com/python-pillow/pillow-depends/main/nasm-2.16.01-win64.zip +- curl -fsSL --ssl-no-revoke -o nasm-win64.zip https://raw.githubusercontent.com/python-pillow/pillow-depends/main/nasm-2.16.01-win64.zip - 7z x nasm-win64.zip -oc:\ - choco install ghostscript --version=10.0.0.20230317 - path c:\nasm-2.16.01;C:\Program Files\gs\gs10.00.0\bin;%PATH% From f4d9c44e3104184a389178b1f80be5bbddad2aac Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 20 Sep 2023 19:24:43 +1000 Subject: [PATCH 6/8] Restrict "params" to list or None --- winbuild/build_prepare.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 34bfaa964f3..a56353e46ae 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -41,14 +41,9 @@ def cmd_rmdir(path: str) -> str: def cmd_nmake( makefile: str | None = None, target: str = "", - params: str | list[str] | tuple[str, ...] = None, + params: list[str] | None = None, ) -> str: - if params is None: - params = "" - elif isinstance(params, (list, tuple)): - params = " ".join(params) - else: - params = str(params) + params = "" if params is None else " ".join(params) return " ".join( [ From 1ba79d2bee268b1ad21881007fb703a230397153 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 20 Sep 2023 14:07:04 +0300 Subject: [PATCH 7/8] Re-instate for/else to avoid a raise after an error and a subsequent success --- winbuild/build_prepare.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 34bfaa964f3..20bb846bc63 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -443,8 +443,7 @@ def download_dep(url: str, file: str) -> None: break except urllib.error.URLError as e: ex = e - - if ex: + else: raise RuntimeError(ex) From 9c754ebab14b3fe4f2609957d2ae7d11fbad23fe Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 21 Sep 2023 12:20:17 +0300 Subject: [PATCH 8/8] Re-remove --ssl-no-revoke --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4546e2a5bb0..cc4d56d0bc9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -21,10 +21,10 @@ environment: install: - '%PYTHON%\%EXECUTABLE% --version' - '%PYTHON%\%EXECUTABLE% -m pip install --upgrade pip' -- curl -fsSL --ssl-no-revoke -o pillow-test-images.zip https://github.com/python-pillow/test-images/archive/main.zip +- curl -fsSL -o pillow-test-images.zip https://github.com/python-pillow/test-images/archive/main.zip - 7z x pillow-test-images.zip -oc:\ - xcopy /S /Y c:\test-images-main\* c:\pillow\tests\images -- curl -fsSL --ssl-no-revoke -o nasm-win64.zip https://raw.githubusercontent.com/python-pillow/pillow-depends/main/nasm-2.16.01-win64.zip +- curl -fsSL -o nasm-win64.zip https://raw.githubusercontent.com/python-pillow/pillow-depends/main/nasm-2.16.01-win64.zip - 7z x nasm-win64.zip -oc:\ - choco install ghostscript --version=10.0.0.20230317 - path c:\nasm-2.16.01;C:\Program Files\gs\gs10.00.0\bin;%PATH%