From 2f180c50c6df9198da692264a477433976ae1731 Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi Date: Tue, 4 Feb 2025 14:26:07 -0800 Subject: [PATCH 1/5] [build.ps1] don't use host Python any more Cherrypick commit https://github.com/swiftlang/swift/pull/76594 --- utils/build.ps1 | 60 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 5463278a6c740..d549468d8dd6a 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -194,14 +194,6 @@ $WiXVersion = "4.0.5" # Avoid $env:ProgramFiles in case this script is running as x86 $UnixToolsBinDir = "$env:SystemDrive\Program Files\Git\usr\bin" -$python = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Shared\Python39_64\python.exe" -if (-not (Test-Path $python)) { - $python = (where.exe python) | Select-Object -First 1 - if (-not (Test-Path $python)) { - throw "Python.exe not found" - } -} - if ($Android -and ($AndroidSDKs.Length -eq 0)) { # Enable all android SDKs by default. $AndroidSDKs = @("aarch64","armv7","i686","x86_64") @@ -358,6 +350,10 @@ function Get-BisonExecutable { return Join-Path -Path $BinaryCache -ChildPath "win_flex_bison\win_bison.exe" } +function Get-PythonExecutable { + return Join-Path -Path $BinaryCache -ChildPath "Python$($HostArch.CMakeName)-$PythonVersion\tools\python.exe" +} + function Get-InstallDir($Arch) { if ($Arch -eq $HostArch) { $ProgramFilesName = "Program Files" @@ -747,10 +743,43 @@ function Fetch-Dependencies { } } + function Ensure-PythonModules($Python) { + # First ensure pip is installed, else bootstrap it + try { + Invoke-Program -OutNull $Python -m pip *> $null + } catch { + Write-Output "Installing pip ..." + Invoke-Program -OutNull $Python '-I' -m ensurepip -U --default-pip + } + # 'packaging' is required for building LLVM 18+ + try { + Invoke-Program -OutNull $Python -c 'import packaging' *> $null + } catch { + $WheelURL = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl" + $WheelHash = "5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" + DownloadAndVerify $WheelURL "$BinaryCache\python\packaging-24.1-py3-none-any.whl" $WheelHash + Write-Output "Installing 'packaging-24.1-py3-none-any.whl' ..." + Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\packaging-24.1-py3-none-any.whl" --disable-pip-version-check + } + # 'setuptools' provides 'distutils' module for Python 3.12+, required for SWIG support + # https://github.com/swiftlang/llvm-project/issues/9289 + try { + Invoke-Program -OutNull $Python -c 'import distutils' *> $null + } catch { + $WheelURL = "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl" + $WheelHash = "35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2" + DownloadAndVerify $WheelURL "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" $WheelHash + Write-Output "Installing 'setuptools-75.1.0-py3-none-any.whl' ..." + Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" --disable-pip-version-check + } + } + Download-Python $HostArchName if ($IsCrossCompiling) { Download-Python $BuildArchName } + # Ensure Python modules that are required as host build tools + Ensure-PythonModules "$(Get-PythonExecutable)" if ($Android) { # Only a specific NDK version is supported right now. @@ -1456,6 +1485,9 @@ function Build-Compilers() { } } + $PythonRoot = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools" + $PythonLibName = "python{0}{1}" -f ([System.Version]$PythonVersion).Major, ([System.Version]$PythonVersion).Minor + # The STL in VS 17.10 requires Clang 17 or higher, but Swift toolchains prior to version 6 include older versions # of Clang. If bootstrapping with an older toolchain, we need to relax to relax this requirement with # ALLOW_COMPILER_AND_STL_VERSION_MISMATCH. @@ -1487,10 +1519,10 @@ function Build-Compilers() { LLVM_NATIVE_TOOL_DIR = $BuildTools; LLVM_TABLEGEN = (Join-Path $BuildTools -ChildPath "llvm-tblgen.exe"); LLVM_USE_HOST_TOOLS = "NO"; - Python3_EXECUTABLE = "$python"; - Python3_INCLUDE_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\include"; - Python3_LIBRARY = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\libs\python39.lib"; - Python3_ROOT_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools"; + Python3_EXECUTABLE = (Get-PythonExecutable); + Python3_INCLUDE_DIR = "$PythonRoot\include"; + Python3_LIBRARY = "$PythonRoot\libs\$PythonLibName.lib"; + Python3_ROOT_DIR = $PythonRoot; SWIFT_BUILD_SWIFT_SYNTAX = "YES"; SWIFT_CLANG_LOCATION = (Get-PinnedToolchainTool); SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY = "YES"; @@ -1815,7 +1847,7 @@ function Build-Runtime([Platform]$Platform, $Arch) { }) } - Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'DEFAULT_USE_RUNTIME': 'MD' } }), encoding='utf-8'))" ` + Invoke-Program "$(Get-PythonExecutable)" -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'DEFAULT_USE_RUNTIME': 'MD' } }), encoding='utf-8'))" ` -OutFile "$($Arch.SDKInstallRoot)\SDKSettings.plist" } @@ -2046,7 +2078,7 @@ function Build-Testing([Platform]$Platform, $Arch, [switch]$Test = $false) { function Write-PlatformInfoPlist($Arch) { $PList = Join-Path -Path $Arch.PlatformInstallRoot -ChildPath "Info.plist" - Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFT_TESTING_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" ` + Invoke-Program "$(Get-PythonExecutable)" -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFT_TESTING_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" ` -OutFile "$PList" } From dd9806c767db159587e2a59bf4071179772945b0 Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi Date: Tue, 4 Feb 2025 15:01:08 -0800 Subject: [PATCH 2/5] build: enable libxml2 in the toolchain, use llvm-mt Cherrypick commit https://github.com/swiftlang/swift/pull/77862 --- cmake/caches/Windows-aarch64.cmake | 6 ++++-- cmake/caches/Windows-x86_64.cmake | 6 ++++-- utils/build.ps1 | 16 ++++++++-------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cmake/caches/Windows-aarch64.cmake b/cmake/caches/Windows-aarch64.cmake index 6dd9a786b9065..089caa349eaa6 100644 --- a/cmake/caches/Windows-aarch64.cmake +++ b/cmake/caches/Windows-aarch64.cmake @@ -50,7 +50,7 @@ set(LLVM_TARGETS_TO_BUILD AArch64 ARM WebAssembly X86 CACHE STRING "") set(LLVM_BUILD_LLVM_DYLIB NO CACHE BOOL "") set(LLVM_BUILD_LLVM_C_DYLIB NO CACHE BOOL "") set(LLVM_ENABLE_LIBEDIT NO CACHE BOOL "") -set(LLVM_ENABLE_LIBXML2 NO CACHE BOOL "") +set(LLVM_ENABLE_LIBXML2 YES CACHE BOOL "") set(LLVM_ENABLE_OCAMLDOC NO CACHE BOOL "") set(LLVM_ENABLE_TERMINFO NO CACHE BOOL "") set(LLVM_ENABLE_Z3_SOLVER NO CACHE BOOL "") @@ -62,12 +62,14 @@ set(LLVM_INCLUDE_GO_TESTS NO CACHE BOOL "") set(LLVM_TOOL_GOLD_BUILD NO CACHE BOOL "") set(LLVM_TOOL_LLVM_SHLIB_BUILD NO CACHE BOOL "") +set(CLANG_ENABLE_LIBXML2 NO CACHE BOOL "") + # Avoid swig dependency for lldb set(LLDB_ALLOW_STATIC_BINDINGS YES CACHE BOOL "") set(LLDB_USE_STATIC_BINDINGS YES CACHE BOOL "") set(LLDB_ENABLE_PYTHON YES CACHE BOOL "") set(LLDB_EMBED_PYTHON_HOME NO CACHE BOOL "") -set(LLDB_ENABLE_LIBXML2 NO CACHE BOOL "") +set(LLDB_ENABLE_LIBXML2 YES CACHE BOOL "") # This requires perl which may not be available on Windows set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "") diff --git a/cmake/caches/Windows-x86_64.cmake b/cmake/caches/Windows-x86_64.cmake index 0f146d3f4e43a..71396be127fa8 100644 --- a/cmake/caches/Windows-x86_64.cmake +++ b/cmake/caches/Windows-x86_64.cmake @@ -90,7 +90,7 @@ set(LLVM_TARGETS_TO_BUILD AArch64 ARM WebAssembly X86 CACHE STRING "") set(LLVM_BUILD_LLVM_DYLIB NO CACHE BOOL "") set(LLVM_BUILD_LLVM_C_DYLIB NO CACHE BOOL "") set(LLVM_ENABLE_LIBEDIT NO CACHE BOOL "") -set(LLVM_ENABLE_LIBXML2 NO CACHE BOOL "") +set(LLVM_ENABLE_LIBXML2 YES CACHE BOOL "") set(LLVM_ENABLE_OCAMLDOC NO CACHE BOOL "") set(LLVM_ENABLE_TERMINFO NO CACHE BOOL "") set(LLVM_ENABLE_Z3_SOLVER NO CACHE BOOL "") @@ -102,12 +102,14 @@ set(LLVM_INCLUDE_GO_TESTS NO CACHE BOOL "") set(LLVM_TOOL_GOLD_BUILD NO CACHE BOOL "") set(LLVM_TOOL_LLVM_SHLIB_BUILD NO CACHE BOOL "") +set(CLANG_ENABLE_LIBXML2 NO CACHE BOOL "") + # Avoid swig dependency for lldb set(LLDB_ALLOW_STATIC_BINDINGS YES CACHE BOOL "") set(LLDB_USE_STATIC_BINDINGS YES CACHE BOOL "") set(LLDB_ENABLE_PYTHON YES CACHE BOOL "") set(LLDB_EMBED_PYTHON_HOME NO CACHE BOOL "") -set(LLDB_ENABLE_LIBXML2 NO CACHE BOOL "") +set(LLDB_ENABLE_LIBXML2 YES CACHE BOOL "") # This requires perl which may not be available on Windows set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "") diff --git a/utils/build.ps1 b/utils/build.ps1 index d549468d8dd6a..2abc996bf1ace 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -988,7 +988,6 @@ function Build-CMakeProject { } TryAdd-KeyValue $Defines CMAKE_BUILD_TYPE Release - TryAdd-KeyValue $Defines CMAKE_MT "mt" $CFlags = @() switch ($Platform) { @@ -1411,6 +1410,8 @@ function Build-BuildTools($Arch) { -BuildTargets llvm-tblgen,clang-tblgen,clang-pseudo-gen,clang-tidy-confusable-chars-gen,lldb-tblgen,llvm-config,swift-def-to-strings-converter,swift-serialize-diagnostics,swift-compatibility-symbols ` -Defines @{ CMAKE_CROSSCOMPILING = "NO"; + CLANG_ENABLE_LIBXML2 = "NO"; + LLDB_ENABLE_LIBXML2 = "NO"; LLDB_ENABLE_PYTHON = "NO"; LLDB_INCLUDE_TESTS = "NO"; LLDB_ENABLE_SWIFT_SUPPORT = "NO"; @@ -1509,7 +1510,9 @@ function Build-Compilers() { -Defines ($TestingDefines + @{ CLANG_TABLEGEN = (Join-Path -Path $BuildTools -ChildPath "clang-tblgen.exe"); CLANG_TIDY_CONFUSABLE_CHARS_GEN = (Join-Path -Path $BuildTools -ChildPath "clang-tidy-confusable-chars-gen.exe"); + CMAKE_FIND_PACKAGE_PREFER_CONFIG = "YES"; CMAKE_Swift_FLAGS = $SwiftFlags; + LibXml2_DIR = "$LibraryRoot\libxml2-2.11.5\usr\lib\Windows\$($Arch.LLVMName)\cmake\libxml2-2.11.5"; LLDB_PYTHON_EXE_RELATIVE_PATH = "python.exe"; LLDB_PYTHON_EXT_SUFFIX = ".pyd"; LLDB_PYTHON_RELATIVE_PATH = "lib/site-packages"; @@ -1932,14 +1935,9 @@ function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { -Defines (@{ ENABLE_TESTING = "NO"; FOUNDATION_BUILD_TOOLS = if ($Platform -eq "Windows") { "YES" } else { "NO" }; + CMAKE_FIND_PACKAGE_PREFER_CONFIG = "YES"; CURL_DIR = "$LibraryRoot\curl-8.9.1\usr\lib\$Platform\$ShortArch\cmake\CURL"; - LIBXML2_LIBRARY = if ($Platform -eq "Windows") { - "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\libxml2s.lib"; - } else { - "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\libxml2.a"; - }; - LIBXML2_INCLUDE_DIR = "$LibraryRoot\libxml2-2.11.5\usr\include\libxml2"; - LIBXML2_DEFINITIONS = "-DLIBXML_STATIC"; + LibXml2_DIR = "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\cmake\libxml2-2.11.5"; ZLIB_LIBRARY = if ($Platform -eq "Windows") { "$LibraryRoot\zlib-1.3.1\usr\lib\$Platform\$ShortArch\zlibstatic.lib" } else { @@ -2778,6 +2776,7 @@ if (-not $SkipBuild) { Invoke-BuildStep Build-CMark $BuildArch Invoke-BuildStep Build-BuildTools $BuildArch if ($IsCrossCompiling) { + Invoke-BuildStep Build-XML2 Windows $BuildArch Invoke-BuildStep Build-Compilers -Build $BuildArch } if ($IncludeDS2) { @@ -2785,6 +2784,7 @@ if (-not $SkipBuild) { } Invoke-BuildStep Build-CMark $HostArch + Invoke-BuildStep Build-XML2 Windows $HostArch Invoke-BuildStep Build-Compilers $HostArch } From 93f6ccb4c0074bdd28a78552d5ca3e2cc80c0238 Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi Date: Tue, 4 Feb 2025 15:03:15 -0800 Subject: [PATCH 3/5] Fix python arch on the windows arm64 CI Cherrypick commit https://github.com/swiftlang/swift/pull/77940 --- utils/build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 2abc996bf1ace..3a3ee5d517c31 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -351,7 +351,7 @@ function Get-BisonExecutable { } function Get-PythonExecutable { - return Join-Path -Path $BinaryCache -ChildPath "Python$($HostArch.CMakeName)-$PythonVersion\tools\python.exe" + return Join-Path -Path $BinaryCache -ChildPath "Python$($BuildArch.CMakeName)-$PythonVersion\tools\python.exe" } function Get-InstallDir($Arch) { From 6559e5a68c04483f8371128572812906350fe86a Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi Date: Tue, 4 Feb 2025 15:05:11 -0800 Subject: [PATCH 4/5] utils: simplify mimalloc build handling Cherrypick commit https://github.com/swiftlang/swift/pull/77986 --- utils/build.ps1 | 83 ++++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 3a3ee5d517c31..55f1a60f1ee9a 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -1556,47 +1556,52 @@ function Build-Mimalloc() { [hashtable]$Arch ) - if ($Arch -eq $ArchX64) { - $Args = @() - Isolate-EnvVars { - Invoke-VsDevShell $Arch - # Avoid hard-coding the VC tools version number - $VCRedistDir = (Get-ChildItem "${env:VCToolsRedistDir}\$($HostArch.ShortName)" -Filter "Microsoft.VC*.CRT").FullName - if ($VCRedistDir) { - $Args += "-p:VCRedistDir=$VCRedistDir\" - } - } - $Args += "$SourceCache\mimalloc\ide\vs2022\mimalloc.sln" - $Args += "-p:Configuration=Release" - $Args += "-p:ProductArchitecture=$($Arch.VSName)" - Invoke-Program $msbuild @Args - $Dest = "$($Arch.ToolchainInstallRoot)\usr\bin" - Copy-Item -Path "$SourceCache\mimalloc\out\msvc-$($Arch.ShortName)\Release\mimalloc-override.dll" ` - -Destination "$Dest" - Copy-Item -Path "$SourceCache\mimalloc\out\msvc-$($Arch.ShortName)\Release\mimalloc-redirect.dll" ` - -Destination "$Dest" - $MimallocExecutables = @("swift.exe","swiftc.exe","swift-driver.exe","swift-frontend.exe") - $MimallocExecutables += @("clang.exe","clang++.exe","clang-cl.exe") - $MimallocExecutables += @("lld.exe","lld-link.exe","ld.lld.exe","ld64.lld.exe") - foreach ($Exe in $MimallocExecutables) { - $ExePath = [IO.Path]::Combine($Dest, $Exe) - # Binary-patch in place - $Args = @() - $Args += "-f" - $Args += "-i" - $Args += "-v" - $Args += $ExePath - Invoke-Program "$SourceCache\mimalloc\bin\minject" @Args - # Log the import table - $Args = @() - $Args += "-l" - $Args += $ExePath - Invoke-Program "$SourceCache\mimalloc\bin\minject" @Args - dir "$ExePath" - } - } else { + if ($Arch -ne $ArchX64) { throw "mimalloc is currently supported for X64 only" } + + $MSBuildArgs = @("$SourceCache\mimalloc\ide\vs2022\mimalloc.sln") + $MSBuildArgs += "-noLogo" + $MSBuildArgs += "-maxCpuCount" + $MSBuildArgs += "-p:Configuration=Release" + $MSBuildArgs += "-p:ProductArchitecture=$($Arch.VSName)" + + Isolate-EnvVars { + Invoke-VsDevShell $Arch + # Avoid hard-coding the VC tools version number + $VCRedistDir = (Get-ChildItem "${env:VCToolsRedistDir}\$($HostArch.ShortName)" -Filter "Microsoft.VC*.CRT").FullName + if ($VCRedistDir) { + $MSBuildArgs += "-p:VCRedistDir=$VCRedistDir\" + } + } + + Invoke-Program $msbuild @MSBuildArgs + + $Products = @( "mimalloc-override.dll", "mimalloc-redirect.dll" ) + foreach ($Product in $Products) { + Copy-Item -Path "$SourceCache\mimalloc\out\msvc-$($Arch.ShortName)\Release\$Product" -Destination "$(Arch.ToolchainInstallRoot)\usr\bin" + } + + $Tools = @( + "swift.exe", + "swiftc.exe", + "swift-driver.exe", + "swift-frontend.exe", + "clang.exe", + "clang++.exe", + "clang-cl.exe", + "lld.exe", + "lld-link.exe", + "ld.lld.exe", + "ld64.lld.exe" + ) + foreach ($Tool in $Tools) { + $Binary = [IO.Path]::Combine($Dest, $Tool) + # Binary-patch in place + Invoke-Program "$SourceCache\mimalloc\bin\minject" @("-f", "-i", "-v", $Binary) + # Log the import table + Invoke-Program "$SourceCache\mimalloc\bin\minject" @("-l", $Binary) + } } function Build-LLVM([Platform]$Platform, $Arch) { From a116fb3a3bfabf029c5e7bcca35478f9bc0af545 Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi Date: Wed, 12 Feb 2025 13:12:21 -0800 Subject: [PATCH 5/5] tools: use LibXml2::LibXml2 to link against Adjust the build rules to use the `LibXml2::LibXml2` target rather than use the explicit include paths and link flags. This allows us to track additional dependencies (implicit linked libraries) as well as properly propagate the include paths and library search paths. Cherrypick commit https://github.com/swiftlang/swift/pull/77848 --- tools/swift-ide-test/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/swift-ide-test/CMakeLists.txt b/tools/swift-ide-test/CMakeLists.txt index 1c8dacdee059c..3cc33b10db1b4 100644 --- a/tools/swift-ide-test/CMakeLists.txt +++ b/tools/swift-ide-test/CMakeLists.txt @@ -17,12 +17,8 @@ target_link_libraries(swift-ide-test # If libxml2 is available, make it available for swift-ide-test. if(LLVM_ENABLE_LIBXML2) - include_directories(SYSTEM ${LIBXML2_INCLUDE_DIR}) - target_link_libraries(swift-ide-test PRIVATE ${LIBXML2_LIBRARIES}) + target_link_libraries(swift-ide-test PRIVATE LibXml2::LibXml2) target_compile_definitions(swift-ide-test PRIVATE SWIFT_HAVE_LIBXML=1) - if("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "OpenBSD" AND NOT CMAKE_CROSSCOMPILING) - include_directories(SYSTEM "/usr/local/include") - endif() endif() # Create a symlink for swift-api-dump.py in the bin directory