diff --git a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py index 4a0ca330301c8..54c80b7ec15e9 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py +++ b/utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py @@ -14,8 +14,8 @@ from . import product from . import wasisysroot -from .swift_testing import SwiftTestingCMakeShim from .wasmstdlib import WasmStdlib, WasmThreadsStdlib +from .cmake_product import CMakeProduct from .. import shell @@ -41,49 +41,156 @@ def should_test(self, host_target): def _target_package_path(self, swift_host_triple): return os.path.join(self.build_dir, 'Toolchains', swift_host_triple) - def _build_swift_testing(self, swift_host_triple, short_triple, wasi_sysroot): - # TODO: We currently build swift-testing outside of SwiftTesting - # build-script product because we build Wasm stdlib outside of - # the main Swift build unit and we can't use build-script's cross - # compilation infrastructure. - # Once stdlib build is decoupled from compiler's CMake build unit - # and we can use different CMake options for different targets - # for stdlib build, we can fully unify library builds with the - # regular path. + def _append_platform_cmake_options(self, cmake_options, swift_host_triple, has_pthread, wasi_sysroot, extra_swift_flags): + cmake_options.define('CMAKE_SYSTEM_NAME:STRING', 'WASI') + cmake_options.define('CMAKE_SYSTEM_PROCESSOR:STRING', 'wasm32') + cmake_options.define('CMAKE_C_COMPILER_TARGET', swift_host_triple) + cmake_options.define('CMAKE_CXX_COMPILER_TARGET', swift_host_triple) + cmake_options.define( + 'CMAKE_Swift_COMPILER_TARGET', swift_host_triple) + cmake_options.define('CMAKE_SYSROOT', wasi_sysroot) + dest_dir = self._target_package_path(swift_host_triple) + swift_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'swift_static') + clang_resource_dir = os.path.join(swift_resource_dir, 'clang') + + swift_flags = ['-sdk', wasi_sysroot, '-resource-dir', + swift_resource_dir] + extra_swift_flags + c_flags = ['-resource-dir', clang_resource_dir] + cxx_flags = c_flags + ['-fno-exceptions'] + if has_pthread: + clang_flags = ['-mthread-model', 'posix', '-pthread'] + c_flags.extend(clang_flags) + cxx_flags.extend(clang_flags) + swift_flags.extend(['-Xcc', '-matomics', '-Xcc', '-mbulk-memory', + '-Xcc', '-mthread-model', '-Xcc', 'posix', '-Xcc', '-pthread']) - swift_testing = SwiftTestingCMakeShim( + cmake_options.define('CMAKE_Swift_FLAGS', ' '.join(swift_flags)) + cmake_options.define('CMAKE_C_FLAGS', ' '.join(c_flags)) + cmake_options.define('CMAKE_CXX_FLAGS', ' '.join(cxx_flags)) + cmake_options.define('CMAKE_Swift_COMPILER_FORCED', 'TRUE') + cmake_options.define('CMAKE_CXX_COMPILER_FORCED', 'TRUE') + cmake_options.define('CMAKE_BUILD_TYPE', self.args.build_variant) + + # Explicitly choose ar and ranlib from just-built LLVM tools since tools in the host system + # unlikely support Wasm object format. + native_toolchain_path = self.native_toolchain_path(self.args.host_target) + cmake_options.define('CMAKE_AR', os.path.join( + native_toolchain_path, 'bin', 'llvm-ar')) + cmake_options.define('CMAKE_RANLIB', os.path.join( + native_toolchain_path, 'bin', 'llvm-ranlib')) + + def _build_libxml2(self, swift_host_triple, has_pthread, wasi_sysroot): + libxml2 = CMakeProduct( args=self.args, toolchain=self.toolchain, source_dir=os.path.join( - os.path.dirname(self.source_dir), 'swift-testing'), - build_dir=os.path.join( - os.path.dirname(self.build_dir), - 'swift-testing-%s' % short_triple)) + os.path.dirname(self.source_dir), 'libxml2'), + build_dir=os.path.join(self.build_dir, 'libxml2', swift_host_triple)) + self._append_platform_cmake_options( + libxml2.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, []) + libxml2.cmake_options.define('LIBXML2_WITH_C14N', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_CATALOG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_DEBUG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_DOCB', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_FTP', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_HTML', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_HTTP', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ICONV', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ICU', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ISO8859X', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_LEGACY', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_LZMA', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_MEM_DEBUG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_MODULES', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_OUTPUT', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_PATTERN', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_PROGRAMS', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_PUSH', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_PYTHON', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_READER', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_REGEXPS', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_RUN_DEBUG', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_SAX1', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_SCHEMAS', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_SCHEMATRON', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_TESTS', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_TREE', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_VALID', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_WRITER', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_XINCLUDE', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_XPATH', 'TRUE') + libxml2.cmake_options.define('LIBXML2_WITH_XPTR', 'FALSE') + libxml2.cmake_options.define('LIBXML2_WITH_ZLIB', 'FALSE') + libxml2.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE') - swift_testing.cmake_options.define('CMAKE_SYSTEM_NAME:STRING', 'WASI') - swift_testing.cmake_options.define('CMAKE_SYSTEM_PROCESSOR:STRING', 'wasm32') - swift_testing.cmake_options.define( - 'CMAKE_CXX_COMPILER_TARGET', swift_host_triple) - swift_testing.cmake_options.define( - 'CMAKE_Swift_COMPILER_TARGET', swift_host_triple) - swift_testing.cmake_options.define('CMAKE_SYSROOT', wasi_sysroot) - swift_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'swift_static') - swift_testing.cmake_options.define( - 'CMAKE_Swift_FLAGS', - f'-sdk {wasi_sysroot} -resource-dir {swift_resource_dir}') - clang_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'clang') + cmake_thread_enabled = 'TRUE' if has_pthread else 'FALSE' + libxml2.cmake_options.define('LIBXML2_WITH_THREAD_ALLOC', cmake_thread_enabled) + libxml2.cmake_options.define('LIBXML2_WITH_THREADS', cmake_thread_enabled) + libxml2.cmake_options.define('HAVE_PTHREAD_H', cmake_thread_enabled) + + libxml2.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + with shell.pushd(libxml2.build_dir): + shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/', '--component', 'development'], + env={'DESTDIR': wasi_sysroot}) + + def _build_foundation(self, swift_host_triple, has_pthread, wasi_sysroot): + source_root = os.path.dirname(self.source_dir) + host_toolchain = self.native_toolchain_path(self.args.host_target) + + foundation = CMakeProduct( + args=self.args, + toolchain=self.toolchain, + source_dir=os.path.join( + os.path.dirname(self.source_dir), 'swift-corelibs-foundation'), + build_dir=os.path.join(self.build_dir, 'foundation', swift_host_triple)) + self._append_platform_cmake_options( + foundation.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, []) + foundation.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE') + foundation.cmake_options.define('FOUNDATION_BUILD_TOOLS', 'FALSE') + foundation.cmake_options.define('_SwiftCollections_SourceDIR', os.path.join(source_root, 'swift-collections')) + foundation.cmake_options.define('_SwiftFoundation_SourceDIR', os.path.join(source_root, 'swift-foundation')) + foundation.cmake_options.define('_SwiftFoundationICU_SourceDIR', os.path.join(source_root, 'swift-foundation-icu')) + foundation.cmake_options.define('SwiftFoundation_MACRO', os.path.join(host_toolchain, 'lib', 'swift', 'host', 'plugins')) + + foundation.cmake_options.define('LIBXML2_INCLUDE_DIR', os.path.join(wasi_sysroot, 'include', 'libxml2')) + foundation.cmake_options.define('LIBXML2_LIBRARY', os.path.join(wasi_sysroot, 'lib')) + + foundation.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + + dest_dir = self._target_package_path(swift_host_triple) + with shell.pushd(foundation.build_dir): + shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/usr'], + env={'DESTDIR': dest_dir}) + + def _build_swift_testing(self, swift_host_triple, has_pthread, wasi_sysroot): + swift_testing = CMakeProduct( + args=self.args, + toolchain=self.toolchain, + source_dir=os.path.join( + os.path.dirname(self.source_dir), 'swift-testing'), + build_dir=os.path.join(self.build_dir, 'swift-testing', swift_host_triple)) + # For statically linked objects in an archive, we have to use singlethreaded + # LLVM codegen unit to prevent runtime metadata sections from being stripped + # at link-time. + self._append_platform_cmake_options( + swift_testing.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, + extra_swift_flags=['-Xfrontend', '-enable-single-module-llvm-emission']) + swift_testing.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE') swift_testing.cmake_options.define( - 'CMAKE_CXX_FLAGS', f'-resource-dir {clang_resource_dir}') - swift_testing.cmake_options.define('CMAKE_Swift_COMPILER_FORCED', 'TRUE') - swift_testing.cmake_options.define('CMAKE_CXX_COMPILER_FORCED', 'TRUE') + 'CMAKE_Swift_COMPILATION_MODE', 'wholemodule') + swift_testing.cmake_options.define('SwiftTesting_MACRO', 'NO') - swift_testing.build('wasi-wasm32') + swift_testing.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + dest_dir = self._target_package_path(swift_host_triple) with shell.pushd(swift_testing.build_dir): shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/usr'], env={'DESTDIR': dest_dir}) - def _build_target_package(self, swift_host_triple, short_triple, + def _build_target_package(self, swift_host_triple, has_pthread, stdlib_build_path, llvm_runtime_libs_build_path, wasi_sysroot): @@ -103,8 +210,11 @@ def _build_target_package(self, swift_host_triple, short_triple, shell.call([self.toolchain.cmake, '--install', '.', '--component', 'clang_rt.builtins-wasm32'], env={'DESTDIR': clang_dir}) + + self._build_libxml2(swift_host_triple, has_pthread, wasi_sysroot) + self._build_foundation(swift_host_triple, has_pthread, wasi_sysroot) # Build swift-testing - self._build_swift_testing(swift_host_triple, short_triple, wasi_sysroot) + self._build_swift_testing(swift_host_triple, has_pthread, wasi_sysroot) return dest_dir @@ -114,26 +224,26 @@ def build(self, host_target): build_root, '%s-%s' % ('wasmllvmruntimelibs', host_target)) target_packages = [] - # NOTE: We have three types of target triples: + # NOTE: We have two types of target triples: # 1. swift_host_triple: The triple used by the Swift compiler's '-target' option # 2. clang_multiarch_triple: The triple used by Clang to find library # and header paths from the sysroot # https://github.com/llvm/llvm-project/blob/73ef397fcba35b7b4239c00bf3e0b4e689ca0add/clang/lib/Driver/ToolChains/WebAssembly.cpp#L29-L36 - # 3. short_triple: The triple used by build-script to name the build directory - for swift_host_triple, clang_multiarch_triple, short_triple, build_basename in [ - ('wasm32-unknown-wasi', 'wasm32-wasi', 'wasi-wasm32', 'wasmstdlib'), - # TODO: Enable threads stdlib once sdk-generator supports multi-target SDK + for swift_host_triple, clang_multiarch_triple, build_basename, build_sdk, has_pthread in [ + ('wasm32-unknown-wasi', 'wasm32-wasi', 'wasmstdlib', True, False), + # TODO: Include p1-threads in the Swift SDK once sdk-generator supports multi-target SDK # ('wasm32-unknown-wasip1-threads', 'wasm32-wasip1-threads', - # 'wasip1-threads-wasm32', 'wasmthreadsstdlib'), + # 'wasmthreadsstdlib', False, True), ]: stdlib_build_path = os.path.join( build_root, '%s-%s' % (build_basename, host_target)) wasi_sysroot = wasisysroot.WASILibc.sysroot_install_path( build_root, clang_multiarch_triple) package_path = self._build_target_package( - swift_host_triple, short_triple, stdlib_build_path, + swift_host_triple, has_pthread, stdlib_build_path, llvm_runtime_libs_build_path, wasi_sysroot) - target_packages.append((swift_host_triple, wasi_sysroot, package_path)) + if build_sdk: + target_packages.append((swift_host_triple, wasi_sysroot, package_path)) swiftc_path = os.path.abspath(self.toolchain.swiftc) toolchain_path = os.path.dirname(os.path.dirname(swiftc_path))