Skip to content

winbuild: Refactor dependency versions into constants #7843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
radarhere merged 9 commits into from
Mar 9, 2024
Merged

winbuild: Refactor dependency versions into constants #7843

Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0cc1cfb
Move global into main() to avoid shadowing
hugovk Feb 29, 2024
fb3cb60
Refactor version numbers into constants
hugovk Feb 29, 2024
0b54676
Refactor constants into dict
hugovk Feb 29, 2024
33d9281
Include SF_PROJECTS in f-strings
hugovk Mar 3, 2024
b3fa754
Don't shadow stdlib names
hugovk Mar 3, 2024
e2c267f
Explicitly import urllib.error to avoid IDE warning
hugovk Mar 3, 2024
71018ba
Remove unused LIBXCB constant
hugovk Mar 5, 2024
35b8030
Simplify rf to f-string
hugovk Mar 7, 2024
9848935
Remove unused pillow_dir
hugovk Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 98 additions & 62 deletions winbuild/build_prepare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,33 @@ def cmd_msbuild(
"ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"},
}

V = {
"BROTLI": "1.1.0",
"FREETYPE": "2.13.2",
"FRIBIDI": "1.0.13",
"HARFBUZZ": "8.3.0",
"JPEGTURBO": "3.0.1",
"LCMS2": "2.16",
"LIBPNG": "1.6.39",
"LIBWEBP": "1.3.2",
"LIBXCB": "1.16",
"OPENJPEG": "2.5.2",
"TIFF": "4.6.0",
"XZ": "5.4.5",
"ZLIB": "1.3",
}
V["LIBPNG_DOTLESS"] = V["LIBPNG"].replace(".", "")
V["LIBPNG_XY"] = "".join(V["LIBPNG"].split(".")[:2])
V["ZLIB_DOTLESS"] = V["ZLIB"].replace(".", "")


# dependencies, listed in order of compilation
DEPS = {
"libjpeg": {
"url": SF_PROJECTS
+ "/libjpeg-turbo/files/3.0.1/libjpeg-turbo-3.0.1.tar.gz/download",
"filename": "libjpeg-turbo-3.0.1.tar.gz",
"dir": "libjpeg-turbo-3.0.1",
"url": SF_PROJECTS + f"/libjpeg-turbo/files/{V['JPEGTURBO']}/"
f"libjpeg-turbo-{V['JPEGTURBO']}.tar.gz/download",
"filename": f"libjpeg-turbo-{V['JPEGTURBO']}.tar.gz",
"dir": f"libjpeg-turbo-{V['JPEGTURBO']}",
"license": ["README.ijg", "LICENSE.md"],
"license_pattern": (
"(LEGAL ISSUES\n============\n\n.+?)\n\nREFERENCES\n=========="
Expand Down Expand Up @@ -143,9 +163,9 @@ def cmd_msbuild(
"bins": ["cjpeg.exe", "djpeg.exe"],
},
"zlib": {
"url": "https://zlib.net/zlib13.zip",
"filename": "zlib13.zip",
"dir": "zlib-1.3",
"url": f"https://zlib.net/zlib{V['ZLIB_DOTLESS']}.zip",
"filename": f"zlib{V['ZLIB_DOTLESS']}.zip",
"dir": f"zlib-{V['ZLIB']}",
"license": "README",
"license_pattern": "Copyright notice:\n\n(.+)$",
"build": [
Expand All @@ -157,9 +177,9 @@ def cmd_msbuild(
"libs": [r"*.lib"],
},
"xz": {
"url": SF_PROJECTS + "/lzmautils/files/xz-5.4.5.tar.gz/download",
"filename": "xz-5.4.5.tar.gz",
"dir": "xz-5.4.5",
"url": SF_PROJECTS + f"/lzmautils/files/xz-{V['XZ']}.tar.gz/download",
"filename": f"xz-{V['XZ']}.tar.gz",
"dir": f"xz-{V['XZ']}",
"license": "COPYING",
"build": [
*cmds_cmake("liblzma", "-DBUILD_SHARED_LIBS:BOOL=OFF"),
Expand All @@ -170,9 +190,9 @@ def cmd_msbuild(
"libs": [r"liblzma.lib"],
},
"libwebp": {
"url": "http://downloads.webmproject.org/releases/webp/libwebp-1.3.2.tar.gz",
"filename": "libwebp-1.3.2.tar.gz",
"dir": "libwebp-1.3.2",
"url": f"http://downloads.webmproject.org/releases/webp/libwebp-{V['LIBWEBP']}.tar.gz",
"filename": f"libwebp-{V['LIBWEBP']}.tar.gz",
"dir": f"libwebp-{V['LIBWEBP']}",
"license": "COPYING",
"patch": {
r"src\enc\picture_csp_enc.c": {
Expand All @@ -192,9 +212,9 @@ def cmd_msbuild(
"libs": [r"libsharpyuv.lib", r"libwebp*.lib"],
},
"libtiff": {
"url": "https://download.osgeo.org/libtiff/tiff-4.6.0.tar.gz",
"filename": "tiff-4.6.0.tar.gz",
"dir": "tiff-4.6.0",
"url": f"https://download.osgeo.org/libtiff/tiff-{V['TIFF']}.tar.gz",
"filename": f"tiff-{V['TIFF']}.tar.gz",
"dir": f"tiff-{V['TIFF']}",
"license": "LICENSE.md",
"patch": {
r"libtiff\tif_lzma.c": {
Expand Down Expand Up @@ -224,21 +244,24 @@ def cmd_msbuild(
"libs": [r"libtiff\*.lib"],
},
"libpng": {
"url": SF_PROJECTS + "/libpng/files/libpng16/1.6.39/lpng1639.zip/download",
"filename": "lpng1639.zip",
"dir": "lpng1639",
"url": SF_PROJECTS + f"/libpng/files/libpng{V['LIBPNG_XY']}/{V['LIBPNG']}/"
f"lpng{V['LIBPNG_DOTLESS']}.zip/download",
"filename": f"lpng{V['LIBPNG_DOTLESS']}.zip",
"dir": f"lpng{V['LIBPNG_DOTLESS']}",
"license": "LICENSE",
"build": [
*cmds_cmake("png_static", "-DPNG_SHARED:BOOL=OFF", "-DPNG_TESTS:BOOL=OFF"),
cmd_copy("libpng16_static.lib", "libpng16.lib"),
cmd_copy(
f"libpng{V['LIBPNG_XY']}_static.lib", f"libpng{V['LIBPNG_XY']}.lib"
),
],
"headers": [r"png*.h"],
"libs": [r"libpng16.lib"],
"libs": [rf"libpng{V['LIBPNG_XY']}.lib"],
},
"brotli": {
"url": "https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz",
"filename": "brotli-1.1.0.tar.gz",
"dir": "brotli-1.1.0",
"url": f"https://github.com/google/brotli/archive/refs/tags/v{V['BROTLI']}.tar.gz",
"filename": f"brotli-{V['BROTLI']}.tar.gz",
"dir": f"brotli-{V['BROTLI']}",
"license": "LICENSE",
"build": [
*cmds_cmake(("brotlicommon", "brotlidec"), "-DBUILD_SHARED_LIBS:BOOL=OFF"),
Expand All @@ -247,9 +270,9 @@ def cmd_msbuild(
"libs": ["*.lib"],
},
"freetype": {
"url": "https://download.savannah.gnu.org/releases/freetype/freetype-2.13.2.tar.gz",
"filename": "freetype-2.13.2.tar.gz",
"dir": "freetype-2.13.2",
"url": f"https://download.savannah.gnu.org/releases/freetype/freetype-{V['FREETYPE']}.tar.gz",
"filename": f"freetype-{V['FREETYPE']}.tar.gz",
"dir": f"freetype-{V['FREETYPE']}",
"license": ["LICENSE.TXT", r"docs\FTL.TXT", r"docs\GPLv2.TXT"],
"patch": {
r"builds\windows\vc2010\freetype.vcxproj": {
Expand All @@ -262,7 +285,7 @@ def cmd_msbuild(
"<UserDefines></UserDefines>": "<UserDefines>FT_CONFIG_OPTION_SYSTEM_ZLIB;FT_CONFIG_OPTION_USE_PNG;FT_CONFIG_OPTION_USE_HARFBUZZ;FT_CONFIG_OPTION_USE_BROTLI</UserDefines>", # noqa: E501
"<UserIncludeDirectories></UserIncludeDirectories>": r"<UserIncludeDirectories>{dir_harfbuzz}\src;{inc_dir}</UserIncludeDirectories>", # noqa: E501
"<UserLibraryDirectories></UserLibraryDirectories>": "<UserLibraryDirectories>{lib_dir}</UserLibraryDirectories>", # noqa: E501
"<UserDependencies></UserDependencies>": "<UserDependencies>zlib.lib;libpng16.lib;brotlicommon.lib;brotlidec.lib</UserDependencies>", # noqa: E501
"<UserDependencies></UserDependencies>": f"<UserDependencies>zlib.lib;libpng{V['LIBPNG_XY']}.lib;brotlicommon.lib;brotlidec.lib</UserDependencies>", # noqa: E501
},
r"src/autofit/afshaper.c": {
# link against harfbuzz.lib
Expand All @@ -282,9 +305,10 @@ def cmd_msbuild(
"libs": [r"objs\{msbuild_arch}\Release Static\freetype.lib"],
},
"lcms2": {
"url": SF_PROJECTS + "/lcms/files/lcms/2.16/lcms2-2.16.tar.gz/download",
"filename": "lcms2-2.16.tar.gz",
"dir": "lcms2-2.16",
"url": SF_PROJECTS
+ f"/lcms/files/lcms/{V['LCMS2']}/lcms2-{V['LCMS2']}.tar.gz/download",
"filename": f"lcms2-{V['LCMS2']}.tar.gz",
"dir": f"lcms2-{V['LCMS2']}",
"license": "LICENSE",
"patch": {
r"Projects\VC2022\lcms2_static\lcms2_static.vcxproj": {
Expand All @@ -308,16 +332,16 @@ def cmd_msbuild(
"libs": [r"Lib\MS\*.lib"],
},
"openjpeg": {
"url": "https://github.com/uclouvain/openjpeg/archive/v2.5.2.tar.gz",
"filename": "openjpeg-2.5.2.tar.gz",
"dir": "openjpeg-2.5.2",
"url": f"https://github.com/uclouvain/openjpeg/archive/v{V['OPENJPEG']}.tar.gz",
"filename": f"openjpeg-{V['OPENJPEG']}.tar.gz",
"dir": f"openjpeg-{V['OPENJPEG']}",
"license": "LICENSE",
"build": [
*cmds_cmake(
"openjp2", "-DBUILD_CODEC:BOOL=OFF", "-DBUILD_SHARED_LIBS:BOOL=OFF"
),
cmd_mkdir(r"{inc_dir}\openjpeg-2.5.2"),
cmd_copy(r"src\lib\openjp2\*.h", r"{inc_dir}\openjpeg-2.5.2"),
cmd_mkdir(rf"{{inc_dir}}\openjpeg-{V['OPENJPEG']}"),
cmd_copy(r"src\lib\openjp2\*.h", rf"{{inc_dir}}\openjpeg-{V['OPENJPEG']}"),
],
"libs": [r"bin\*.lib"],
},
Expand All @@ -343,9 +367,9 @@ def cmd_msbuild(
"libs": [r"imagequant.lib"],
},
"harfbuzz": {
"url": "https://github.com/harfbuzz/harfbuzz/archive/8.3.0.zip",
"filename": "harfbuzz-8.3.0.zip",
"dir": "harfbuzz-8.3.0",
"url": f"https://github.com/harfbuzz/harfbuzz/archive/{V['HARFBUZZ']}.zip",
"filename": f"harfbuzz-{V['HARFBUZZ']}.zip",
"dir": f"harfbuzz-{V['HARFBUZZ']}",
"license": "COPYING",
"build": [
*cmds_cmake(
Expand All @@ -358,12 +382,12 @@ def cmd_msbuild(
"libs": [r"*.lib"],
},
"fribidi": {
"url": "https://github.com/fribidi/fribidi/archive/v1.0.13.zip",
"filename": "fribidi-1.0.13.zip",
"dir": "fribidi-1.0.13",
"url": f"https://github.com/fribidi/fribidi/archive/v{V['FRIBIDI']}.zip",
"filename": f"fribidi-{V['FRIBIDI']}.zip",
"dir": f"fribidi-{V['FRIBIDI']}",
"license": "COPYING",
"build": [
cmd_copy(r"COPYING", r"{bin_dir}\fribidi-1.0.13-COPYING"),
cmd_copy(r"COPYING", rf"{{bin_dir}}\fribidi-{V['FRIBIDI']}-COPYING"),
cmd_copy(r"{winbuild_dir}\fribidi.cmake", r"CMakeLists.txt"),
# generated tab.i files cannot be cross-compiled
" ^&^& ".join(
Expand Down Expand Up @@ -456,11 +480,14 @@ def download_dep(url: str, file: str) -> None:
raise RuntimeError(ex)


def extract_dep(url: str, filename: str) -> None:
def extract_dep(url: str, filename: str, prefs: dict[str, str]) -> None:
import tarfile
import zipfile

file = os.path.join(args.depends_dir, filename)
depends_dir = prefs["depends_dir"]
sources_dir = prefs["src_dir"]

file = os.path.join(depends_dir, filename)
if not os.path.exists(file):
# First try our mirror
mirror_url = (
Expand Down Expand Up @@ -499,13 +526,15 @@ def extract_dep(url: str, filename: str) -> None:
raise RuntimeError(msg)


def write_script(name: str, lines: list[str]) -> None:
name = os.path.join(args.build_dir, name)
def write_script(
name: str, lines: list[str], prefs: dict[str, str], verbose: bool
) -> None:
name = os.path.join(prefs["build_dir"], name)
lines = [line.format(**prefs) for line in lines]
print("Writing " + name)
with open(name, "w", newline="") as f:
f.write(os.linesep.join(lines))
if args.verbose:
if verbose:
for line in lines:
print(" " + line)

Expand All @@ -521,7 +550,7 @@ def get_footer(dep: dict) -> list[str]:
return lines


def build_env() -> None:
def build_env(prefs: dict[str, str], verbose: bool) -> None:
lines = [
"if defined DISTUTILS_USE_SDK goto end",
cmd_set("INCLUDE", "{inc_dir}"),
Expand All @@ -534,15 +563,17 @@ def build_env() -> None:
":end",
"@echo on",
]
write_script("build_env.cmd", lines)
write_script("build_env.cmd", lines, prefs, verbose)


def build_dep(name: str) -> str:
def build_dep(name: str, prefs: dict[str, str], verbose: bool) -> str:
dep = DEPS[name]
dir = dep["dir"]
file = f"build_dep_{name}.cmd"
license_dir = prefs["license_dir"]
sources_dir = prefs["src_dir"]

extract_dep(dep["url"], dep["filename"])
extract_dep(dep["url"], dep["filename"], prefs)

licenses = dep["license"]
if isinstance(licenses, str):
Expand Down Expand Up @@ -583,19 +614,19 @@ def build_dep(name: str) -> str:
*get_footer(dep),
]

write_script(file, lines)
write_script(file, lines, prefs, verbose)
return file


def build_dep_all() -> None:
def build_dep_all(disabled: list[str], prefs: dict[str, str], verbose: bool) -> None:
lines = [r'call "{build_dir}\build_env.cmd"']
gha_groups = "GITHUB_ACTIONS" in os.environ
for dep_name in DEPS:
print()
if dep_name in disabled:
print(f"Skipping disabled dependency {dep_name}")
continue
script = build_dep(dep_name)
script = build_dep(dep_name, prefs, verbose)
if gha_groups:
lines.append(f"@echo ::group::Running {script}")
lines.append(rf'cmd.exe /c "{{build_dir}}\{script}"')
Expand All @@ -604,10 +635,10 @@ def build_dep_all() -> None:
lines.append("@echo ::endgroup::")
print()
lines.append("@echo All Pillow dependencies built successfully!")
write_script("build_dep_all.cmd", lines)
write_script("build_dep_all.cmd", lines, prefs, verbose)


if __name__ == "__main__":
def main() -> None:
winbuild_dir = os.path.dirname(os.path.realpath(__file__))
pillow_dir = os.path.realpath(os.path.join(winbuild_dir, ".."))

Expand Down Expand Up @@ -718,12 +749,13 @@ def build_dep_all() -> None:
"pillow_dir": pillow_dir,
"winbuild_dir": winbuild_dir,
# Build paths
"bin_dir": bin_dir,
"build_dir": args.build_dir,
"depends_dir": args.depends_dir,
"inc_dir": inc_dir,
"lib_dir": lib_dir,
"bin_dir": bin_dir,
"src_dir": sources_dir,
"license_dir": license_dir,
"src_dir": sources_dir,
# Compilers / Tools
**msvs,
"cmake": "cmake.exe", # TODO find CMAKE automatically
Expand All @@ -736,6 +768,10 @@ def build_dep_all() -> None:

print()

write_script(".gitignore", ["*"])
build_env()
build_dep_all()
write_script(".gitignore", ["*"], prefs, args.verbose)
build_env(prefs, args.verbose)
build_dep_all(disabled, prefs, args.verbose)


if __name__ == "__main__":
main()