From e93e3af6f9bc5c99d226da9fd9b8ea148d7a4284 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 9 Jun 2025 21:39:17 +0000 Subject: [PATCH 1/5] [wasm] Emit libTesting.a objects with single-threaded LLVM codegen unit Since 5f2b0022d14d6eaddd48d4f36034c31218965ead, swift-testing is being compiled with WMO, which removes some of inter-object references in object files by DCE. The inter-object reference removal revealed a long-standing issue that the runtime metadata sections of objects in an archive are not always included in the final binary if symbols from those objects are not referenced anywhere. To force including all metadata sections in the final binary, we have to emit everything in a single object file when building the archive. This issue happens only for Wasm SDK, which ships swift-testing as a static archive. --- .../swift_build_support/products/wasmswiftsdk.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py index 4a0ca330301c8..71d537a14ccdc 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py +++ b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py @@ -69,9 +69,12 @@ def _build_swift_testing(self, swift_host_triple, short_triple, wasi_sysroot): 'CMAKE_Swift_COMPILER_TARGET', swift_host_triple) swift_testing.cmake_options.define('CMAKE_SYSROOT', wasi_sysroot) swift_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'swift_static') + # For statically linked objects in an archive, we have to use singlethreaded + # LLVM codegen unit to prevent runtime metadata sections from being stripped + # at link-time. swift_testing.cmake_options.define( 'CMAKE_Swift_FLAGS', - f'-sdk {wasi_sysroot} -resource-dir {swift_resource_dir}') + f'-sdk {wasi_sysroot} -resource-dir {swift_resource_dir} -Xfrontend -enable-single-module-llvm-emission') clang_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'clang') swift_testing.cmake_options.define( 'CMAKE_CXX_FLAGS', f'-resource-dir {clang_resource_dir}') From d997d1f54cd960791946aaf4cce208b760077107 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 11 Jun 2025 16:05:40 +0900 Subject: [PATCH 2/5] [wasm] Enable building swift-testing for wasm32-unknown-wasip1-threads --- .../products/wasmswiftsdk.py | 105 +++++++++++------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py index 71d537a14ccdc..cef7917fa70d8 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py +++ b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py @@ -14,8 +14,8 @@ from . import product from . import wasisysroot -from .swift_testing import SwiftTestingCMakeShim from .wasmstdlib import WasmStdlib, WasmThreadsStdlib +from .cmake_product import CMakeProduct from .. import shell @@ -41,52 +41,71 @@ def should_test(self, host_target): def _target_package_path(self, swift_host_triple): return os.path.join(self.build_dir, 'Toolchains', swift_host_triple) - def _build_swift_testing(self, swift_host_triple, short_triple, wasi_sysroot): - # TODO: We currently build swift-testing outside of SwiftTesting - # build-script product because we build Wasm stdlib outside of - # the main Swift build unit and we can't use build-script's cross - # compilation infrastructure. - # Once stdlib build is decoupled from compiler's CMake build unit - # and we can use different CMake options for different targets - # for stdlib build, we can fully unify library builds with the - # regular path. - dest_dir = self._target_package_path(swift_host_triple) + def _append_platform_cmake_options(self, cmake_options, swift_host_triple, has_pthread, wasi_sysroot, extra_swift_flags): + cmake_options.define('CMAKE_SYSTEM_NAME:STRING', 'WASI') + cmake_options.define('CMAKE_SYSTEM_PROCESSOR:STRING', 'wasm32') + cmake_options.define('CMAKE_C_COMPILER_TARGET', swift_host_triple) + cmake_options.define('CMAKE_CXX_COMPILER_TARGET', swift_host_triple) + cmake_options.define( + 'CMAKE_Swift_COMPILER_TARGET', swift_host_triple) + cmake_options.define('CMAKE_SYSROOT', wasi_sysroot) - swift_testing = SwiftTestingCMakeShim( + dest_dir = self._target_package_path(swift_host_triple) + swift_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'swift_static') + clang_resource_dir = os.path.join(swift_resource_dir, 'clang') + + swift_flags = ['-sdk', wasi_sysroot, '-resource-dir', + swift_resource_dir] + extra_swift_flags + c_flags = ['-resource-dir', clang_resource_dir] + cxx_flags = c_flags + ['-fno-exceptions'] + if has_pthread: + clang_flags = ['-mthread-model', 'posix', '-pthread'] + c_flags.extend(clang_flags) + cxx_flags.extend(clang_flags) + swift_flags.extend(['-Xcc', '-matomics', '-Xcc', '-mbulk-memory', + '-Xcc', '-mthread-model', '-Xcc', 'posix', '-Xcc', '-pthread']) + + cmake_options.define('CMAKE_Swift_FLAGS', ' '.join(swift_flags)) + cmake_options.define('CMAKE_C_FLAGS', ' '.join(c_flags)) + cmake_options.define('CMAKE_CXX_FLAGS', ' '.join(cxx_flags)) + cmake_options.define('CMAKE_Swift_COMPILER_FORCED', 'TRUE') + cmake_options.define('CMAKE_CXX_COMPILER_FORCED', 'TRUE') + cmake_options.define('CMAKE_BUILD_TYPE', self.args.build_variant) + + # Explicitly choose ar and ranlib from just-built LLVM tools since tools in the host system + # unlikely support Wasm object format. + native_toolchain_path = self.native_toolchain_path(self.args.host_target) + cmake_options.define('CMAKE_AR', os.path.join( + native_toolchain_path, 'bin', 'llvm-ar')) + cmake_options.define('CMAKE_RANLIB', os.path.join( + native_toolchain_path, 'bin', 'llvm-ranlib')) + + def _build_swift_testing(self, swift_host_triple, has_pthread, wasi_sysroot): + swift_testing = CMakeProduct( args=self.args, toolchain=self.toolchain, source_dir=os.path.join( os.path.dirname(self.source_dir), 'swift-testing'), - build_dir=os.path.join( - os.path.dirname(self.build_dir), - 'swift-testing-%s' % short_triple)) - - swift_testing.cmake_options.define('CMAKE_SYSTEM_NAME:STRING', 'WASI') - swift_testing.cmake_options.define('CMAKE_SYSTEM_PROCESSOR:STRING', 'wasm32') - swift_testing.cmake_options.define( - 'CMAKE_CXX_COMPILER_TARGET', swift_host_triple) - swift_testing.cmake_options.define( - 'CMAKE_Swift_COMPILER_TARGET', swift_host_triple) - swift_testing.cmake_options.define('CMAKE_SYSROOT', wasi_sysroot) - swift_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'swift_static') + build_dir=os.path.join(self.build_dir, 'swift-testing', swift_host_triple)) # For statically linked objects in an archive, we have to use singlethreaded # LLVM codegen unit to prevent runtime metadata sections from being stripped # at link-time. + self._append_platform_cmake_options( + swift_testing.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, + extra_swift_flags=['-Xfrontend', '-enable-single-module-llvm-emission']) + swift_testing.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE') swift_testing.cmake_options.define( - 'CMAKE_Swift_FLAGS', - f'-sdk {wasi_sysroot} -resource-dir {swift_resource_dir} -Xfrontend -enable-single-module-llvm-emission') - clang_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'clang') - swift_testing.cmake_options.define( - 'CMAKE_CXX_FLAGS', f'-resource-dir {clang_resource_dir}') - swift_testing.cmake_options.define('CMAKE_Swift_COMPILER_FORCED', 'TRUE') - swift_testing.cmake_options.define('CMAKE_CXX_COMPILER_FORCED', 'TRUE') + 'CMAKE_Swift_COMPILATION_MODE', 'wholemodule') + swift_testing.cmake_options.define('SwiftTesting_MACRO', 'NO') - swift_testing.build('wasi-wasm32') + swift_testing.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + dest_dir = self._target_package_path(swift_host_triple) with shell.pushd(swift_testing.build_dir): shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/usr'], env={'DESTDIR': dest_dir}) - def _build_target_package(self, swift_host_triple, short_triple, + def _build_target_package(self, swift_host_triple, has_pthread, stdlib_build_path, llvm_runtime_libs_build_path, wasi_sysroot): @@ -107,7 +126,7 @@ def _build_target_package(self, swift_host_triple, short_triple, '--component', 'clang_rt.builtins-wasm32'], env={'DESTDIR': clang_dir}) # Build swift-testing - self._build_swift_testing(swift_host_triple, short_triple, wasi_sysroot) + self._build_swift_testing(swift_host_triple, has_pthread, wasi_sysroot) return dest_dir @@ -117,26 +136,26 @@ def build(self, host_target): build_root, '%s-%s' % ('wasmllvmruntimelibs', host_target)) target_packages = [] - # NOTE: We have three types of target triples: + # NOTE: We have two types of target triples: # 1. swift_host_triple: The triple used by the Swift compiler's '-target' option # 2. clang_multiarch_triple: The triple used by Clang to find library # and header paths from the sysroot # https://github.com/llvm/llvm-project/blob/73ef397fcba35b7b4239c00bf3e0b4e689ca0add/clang/lib/Driver/ToolChains/WebAssembly.cpp#L29-L36 - # 3. short_triple: The triple used by build-script to name the build directory - for swift_host_triple, clang_multiarch_triple, short_triple, build_basename in [ - ('wasm32-unknown-wasi', 'wasm32-wasi', 'wasi-wasm32', 'wasmstdlib'), - # TODO: Enable threads stdlib once sdk-generator supports multi-target SDK - # ('wasm32-unknown-wasip1-threads', 'wasm32-wasip1-threads', - # 'wasip1-threads-wasm32', 'wasmthreadsstdlib'), + for swift_host_triple, clang_multiarch_triple, build_basename, build_sdk, has_pthread in [ + ('wasm32-unknown-wasi', 'wasm32-wasi', 'wasmstdlib', True, False), + # TODO: Include p1-threads in the Swift SDK once sdk-generator supports multi-target SDK + ('wasm32-unknown-wasip1-threads', 'wasm32-wasip1-threads', + 'wasmthreadsstdlib', False, True), ]: stdlib_build_path = os.path.join( build_root, '%s-%s' % (build_basename, host_target)) wasi_sysroot = wasisysroot.WASILibc.sysroot_install_path( build_root, clang_multiarch_triple) package_path = self._build_target_package( - swift_host_triple, short_triple, stdlib_build_path, + swift_host_triple, has_pthread, stdlib_build_path, llvm_runtime_libs_build_path, wasi_sysroot) - target_packages.append((swift_host_triple, wasi_sysroot, package_path)) + if build_sdk: + target_packages.append((swift_host_triple, wasi_sysroot, package_path)) swiftc_path = os.path.abspath(self.toolchain.swiftc) toolchain_path = os.path.dirname(os.path.dirname(swiftc_path)) From afc1c164f8517736fc183b7c2e94bfb15af45fc3 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 18 Jun 2025 10:21:54 +0900 Subject: [PATCH 3/5] Revert "Revert "[wasm] Build and install libxml2 for Wasm SDK"" This reverts commit d5abfa431b2bdcf22df95f0f1e1a045d518c3c3e. --- .../products/wasmswiftsdk.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py index cef7917fa70d8..fe3e195e45889 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py +++ b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py @@ -80,6 +80,61 @@ def _append_platform_cmake_options(self, cmake_options, swift_host_triple, has_p cmake_options.define('CMAKE_RANLIB', os.path.join( native_toolchain_path, 'bin', 'llvm-ranlib')) + def _build_libxml2(self, swift_host_triple, has_pthread, wasi_sysroot): + libxml2 = CMakeProduct( + args=self.args, + toolchain=self.toolchain, + source_dir=os.path.join( + os.path.dirname(self.source_dir), 'libxml2'), + build_dir=os.path.join(self.build_dir, 'libxml2', swift_host_triple)) + self._append_platform_cmake_options( + libxml2.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, []) + libxml2.cmake_options.define('LIBXML2_WITH_C14N', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_CATALOG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_DEBUG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_DOCB', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_FTP', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_HTML', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_HTTP', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ICONV', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ICU', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ISO8859X', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_LEGACY', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_LZMA', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_MEM_DEBUG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_MODULES', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_OUTPUT', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_PATTERN', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_PROGRAMS', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_PUSH', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_PYTHON', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_READER', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_REGEXPS', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_RUN_DEBUG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_SAX1', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_SCHEMAS', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_SCHEMATRON', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_TESTS', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_TREE', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_VALID', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_WRITER', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_XINCLUDE', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_XPATH', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_XPTR', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ZLIB', 'FALSE') + libxml2.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE') + + cmake_thread_enabled = 'TRUE' if has_pthread else 'FALSE' + libxml2.cmake_options.define('LIBXML2_WITH_THREAD_ALLOC', cmake_thread_enabled) + libxml2.cmake_options.define('LIBXML2_WITH_THREADS', cmake_thread_enabled) + libxml2.cmake_options.define('HAVE_PTHREAD_H', cmake_thread_enabled) + + libxml2.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + with shell.pushd(libxml2.build_dir): + shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/', '--component', 'development'], + env={'DESTDIR': wasi_sysroot}) + def _build_swift_testing(self, swift_host_triple, has_pthread, wasi_sysroot): swift_testing = CMakeProduct( args=self.args, @@ -125,6 +180,8 @@ def _build_target_package(self, swift_host_triple, has_pthread, shell.call([self.toolchain.cmake, '--install', '.', '--component', 'clang_rt.builtins-wasm32'], env={'DESTDIR': clang_dir}) + + self._build_libxml2(swift_host_triple, has_pthread, wasi_sysroot) # Build swift-testing self._build_swift_testing(swift_host_triple, has_pthread, wasi_sysroot) From f6eb58fbf2567608a47e4fc25c7d342689883847 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Tue, 17 Jun 2025 09:24:38 +0900 Subject: [PATCH 4/5] [wasm] Build and install Foundation for Wasm SDK --- .../products/wasmswiftsdk.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py index fe3e195e45889..26ad37f5d58f0 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py +++ b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py @@ -135,6 +135,36 @@ def _build_libxml2(self, swift_host_triple, has_pthread, wasi_sysroot): shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/', '--component', 'development'], env={'DESTDIR': wasi_sysroot}) + def _build_foundation(self, swift_host_triple, has_pthread, wasi_sysroot): + source_root = os.path.dirname(self.source_dir) + host_toolchain = self.native_toolchain_path(self.args.host_target) + + foundation = CMakeProduct( + args=self.args, + toolchain=self.toolchain, + source_dir=os.path.join( + os.path.dirname(self.source_dir), 'swift-corelibs-foundation'), + build_dir=os.path.join(self.build_dir, 'foundation', swift_host_triple)) + self._append_platform_cmake_options( + foundation.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, []) + foundation.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE') + foundation.cmake_options.define('FOUNDATION_BUILD_TOOLS', 'FALSE') + foundation.cmake_options.define('_SwiftCollections_SourceDIR', os.path.join(source_root, 'swift-collections')) + foundation.cmake_options.define('_SwiftFoundation_SourceDIR', os.path.join(source_root, 'swift-foundation')) + foundation.cmake_options.define('_SwiftFoundationICU_SourceDIR', os.path.join(source_root, 'swift-foundation-icu')) + foundation.cmake_options.define('SwiftFoundation_MACRO', os.path.join(host_toolchain, 'lib', 'swift', 'host', 'plugins')) + + foundation.cmake_options.define('LIBXML2_INCLUDE_DIR', os.path.join(wasi_sysroot, 'include', 'libxml2')) + foundation.cmake_options.define('LIBXML2_LIBRARY', os.path.join(wasi_sysroot, 'lib')) + + foundation.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + + dest_dir = self._target_package_path(swift_host_triple) + with shell.pushd(foundation.build_dir): + shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/usr'], + env={'DESTDIR': dest_dir}) + def _build_swift_testing(self, swift_host_triple, has_pthread, wasi_sysroot): swift_testing = CMakeProduct( args=self.args, @@ -182,6 +212,7 @@ def _build_target_package(self, swift_host_triple, has_pthread, env={'DESTDIR': clang_dir}) self._build_libxml2(swift_host_triple, has_pthread, wasi_sysroot) + self._build_foundation(swift_host_triple, has_pthread, wasi_sysroot) # Build swift-testing self._build_swift_testing(swift_host_triple, has_pthread, wasi_sysroot) From 05b501f9fb647607bae5ee6a9d5a1ee37a2b0743 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 16 Jul 2025 20:08:09 +0100 Subject: [PATCH 5/5] Disable `wasm32-wasip1-threads` as not buildable on macOS --- .../swift_build_support/products/wasmswiftsdk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py index 26ad37f5d58f0..54c80b7ec15e9 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py +++ b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py @@ -232,8 +232,8 @@ def build(self, host_target): for swift_host_triple, clang_multiarch_triple, build_basename, build_sdk, has_pthread in [ ('wasm32-unknown-wasi', 'wasm32-wasi', 'wasmstdlib', True, False), # TODO: Include p1-threads in the Swift SDK once sdk-generator supports multi-target SDK - ('wasm32-unknown-wasip1-threads', 'wasm32-wasip1-threads', - 'wasmthreadsstdlib', False, True), + # ('wasm32-unknown-wasip1-threads', 'wasm32-wasip1-threads', + # 'wasmthreadsstdlib', False, True), ]: stdlib_build_path = os.path.join( build_root, '%s-%s' % (build_basename, host_target))