macos: initial support for macOS

Various features aren't yet implemented. But it should be "good enough"
to facilitate experimentation.

.editorconfig

@@ -0,0 +1,5 @@
+[Makefile]
+indent_style = tab
+
+[Makefile.extra]
+indent_style = tab

README.rst

@@ -27,7 +27,6 @@ common system libraries.
 Planned features include:
 
 * Support for Windows
-* Support for macOS
 * Static/dynamic linking toggles for dependencies
 
 Instructions
@@ -37,6 +36,10 @@ To build a Python distribution for Linux x64::
 
     $ ./build-linux.py
 
+To build a Python distribution for macOS::
+
+    $ ./build-macos.py
+
 Requirements
 ============
 
@@ -48,6 +51,12 @@ available. The execution environment must have access to a Docker
 daemon (all build operations are performed in Docker containers for
 isolation from the host system).
 
+macOS
+-----
+
+The XCode command line tools must be installed. A Python 3 interpreter
+is required to execute the build. ``/usr/bin/clang`` must exist.
+
 How It Works
 ============
 

build-macos.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import datetime
+import os
+import pathlib
+import subprocess
+import sys
+import venv
+
+
+ROOT = pathlib.Path(os.path.abspath(__file__)).parent
+BUILD = ROOT / 'build'
+DIST = ROOT / 'dist'
+VENV = BUILD / 'venv'
+PIP = VENV / 'bin' / 'pip'
+PYTHON = VENV / 'bin' / 'python'
+REQUIREMENTS = ROOT / 'requirements.txt'
+MAKE_DIR = ROOT / 'cpython-macos'
+
+
+def bootstrap():
+    BUILD.mkdir(exist_ok=True)
+    DIST.mkdir(exist_ok=True)
+
+    venv.create(VENV, with_pip=True)
+
+    subprocess.run([str(PIP), 'install', '-r', str(REQUIREMENTS)],
+                   check=True)
+
+    os.environ['PYBUILD_BOOTSTRAPPED'] = '1'
+    os.environ['PATH'] = '%s:%s' % (str(VENV / 'bin'), os.environ['PATH'])
+    os.environ['PYTHONPATH'] = str(ROOT)
+    subprocess.run([str(PYTHON), __file__], check=True)
+
+
+def run():
+    import zstandard
+    from pythonbuild.downloads import DOWNLOADS
+
+    now = datetime.datetime.utcnow()
+
+    subprocess.run(['make'],
+                   cwd=str(MAKE_DIR), check=True)
+
+    source_path = BUILD / 'cpython-macos.tar'
+    dest_path = DIST / ('cpython-%s-macos-%s.tar.zst' % (
+        DOWNLOADS['cpython-3.7']['version'], now.strftime('%Y%m%dT%H%M')))
+
+    print('compressing Python archive to %s' % dest_path)
+    with source_path.open('rb') as ifh, dest_path.open('wb') as ofh:
+        cctx = zstandard.ZstdCompressor(level=15)
+        cctx.copy_stream(ifh, ofh, source_path.stat().st_size)
+
+
+if __name__ == '__main__':
+    try:
+        if 'PYBUILD_BOOTSTRAPPED' not in os.environ:
+            bootstrap()
+        else:
+            run()
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)

cpython-macos/Makefile

@@ -0,0 +1,50 @@
+ROOT := $(abspath $(CURDIR)/..)
+HERE := $(ROOT)/cpython-macos
+OUTDIR := $(ROOT)/build
+
+BUILD := $(HERE)/build.py
+NULL :=
+
+PLATFORM := macos
+
+default: $(OUTDIR)/cpython-macos.tar
+
+$(OUTDIR)/bzip2-macos.tar: $(HERE)/build-bzip2.sh
+	$(BUILD) bzip2
+
+$(OUTDIR)/libffi-macos.tar: $(HERE)/build-libffi.sh
+	$(BUILD) libffi
+
+$(OUTDIR)/openssl-macos.tar: $(HERE)/build-openssl.sh
+	$(BUILD) openssl
+
+$(OUTDIR)/ncurses-macos.tar: $(HERE)/build-ncurses.sh
+	$(BUILD) ncurses
+
+$(OUTDIR)/sqlite-macos.tar: $(HERE)/build-sqlite.sh
+	$(BUILD) sqlite
+
+$(OUTDIR)/uuid-macos.tar: $(HERE)/build-uuid.sh
+	$(BUILD) uuid
+
+$(OUTDIR)/xz-macos.tar: $(HERE)/build-xz.sh
+	$(BUILD) xz
+
+$(OUTDIR)/zlib-macos.tar: $(HERE)/build-zlib.sh
+	$(BUILD) zlib
+
+PYTHON_DEPENDS := \
+	$(HERE)/static-modules \
+	$(HERE)/build-cpython.sh \
+    $(OUTDIR)/bzip2-macos.tar \
+    $(OUTDIR)/libffi-macos.tar \
+    $(OUTDIR)/ncurses-macos.tar \
+    $(OUTDIR)/openssl-macos.tar \
+    $(OUTDIR)/sqlite-macos.tar \
+    $(OUTDIR)/uuid-macos.tar \
+    $(OUTDIR)/xz-macos.tar \
+    $(OUTDIR)/zlib-macos.tar \
+    $(NULL)
+
+$(OUTDIR)/cpython-macos.tar: $(PYTHON_DEPENDS)
+	$(BUILD) cpython

cpython-macos/build-bzip2.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+tar -xf bzip2-${BZIP2_VERSION}.tar.gz
+
+pushd bzip2-${BZIP2_VERSION}
+
+make -j ${NUM_CPUS} install \
+    CC=clang \
+    CFLAGS="-fPIC" \
+    PREFIX=${ROOT}/out

cpython-macos/build-cpython.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+DEPS_DIR="${ROOT}/deps"
+
+export CC=clang
+export CXX=clang++
+
+# We force linking of external static libraries by removing the shared
+# libraries. This is hacky. But we're building in a temporary directory
+# and it gets the job done.
+find ${DEPS_DIR} -name '*.so*' -exec rm {} \;
+find ${DEPS_DIR} -name '*.dylib' -exec rm {} \;
+
+cat Python-${PYTHON_VERSION}/Modules/Setup.local
+
+pushd Python-${PYTHON_VERSION}
+
+# Most bits look at CFLAGS. But setup.py only looks at CPPFLAGS.
+# So we need to set both.
+CFLAGS="-I${DEPS_DIR}/include -I${DEPS_DIR}/lib/libffi-3.2.1/include -I/${DEPS_DIR}/include/ncurses -I${DEPS_DIR}/include/uuid"
+CPPFLAGS=$CFLAGS
+LDFLAGS="-L${DEPS_DIR}/lib"
+
+CONFIGURE_FLAGS="--prefix=/install --with-openssl=${DEPS_DIR} --without-ensurepip"
+
+if [ -n "${CPYTHON_OPTIMIZED}" ]; then
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-optimizations --enable-lto"
+fi
+
+CFLAGS=$CFLAGS CPPFLAGS=$CFLAGS LDFLAGS=$LDFLAGS \
+    ./configure ${CONFIGURE_FLAGS}
+
+# Supplement produced Makefile with our modifications.
+cat ../Makefile.extra >> Makefile
+
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out/python
+
+# Also copy object files so they can be linked in a custom manner by
+# downstream consumers.
+for d in Modules Objects Parser Programs Python; do
+    mkdir -p ${ROOT}/out/python/build/$d
+    cp -av $d/*.o ${ROOT}/out/python/build/$d/
+done
+
+# config.c defines _PyImport_Inittab and extern references to modules, which
+# downstream consumers may want to strip. We bundle config.c and config.c.in so
+# a custom one can be produced downstream.
+# frozen.c is something similar for frozen modules.
+cp -av Modules/config.c ${ROOT}/out/python/build/Modules/
+cp -av Modules/config.c.in ${ROOT}/out/python/build/Modules/
+cp -av Python/frozen.c ${ROOT}/out/python/build/Python/
+cp ${ROOT}/python-licenses.rst ${ROOT}/out/python/LICENSE.rst

cpython-macos/build-libffi.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export CC=clang
+export CXX=clang++
+
+tar -xf libffi-${LIBFFI_VERSION}.tar.gz
+
+pushd libffi-${LIBFFI_VERSION}
+
+CFLAGS="-fPIC" CPPFLAGS="-fPIC" ./configure \
+    --prefix=/
+
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out

cpython-macos/build-ncurses.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export CC=clang
+export CXX=clang++
+
+tar -xf ncurses-${NCURSES_VERSION}.tar.gz
+
+pushd ncurses-${NCURSES_VERSION}
+
+CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" ./configure \
+    --prefix=/
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out

cpython-macos/build-openssl.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export CC=clang
+export CXX=clang++
+
+tar -xf openssl-${OPENSSL_VERSION}.tar.gz
+
+pushd openssl-${OPENSSL_VERSION}
+
+/usr/bin/perl ./Configure --prefix=/ darwin64-x86_64-cc
+
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out

cpython-macos/build-sqlite.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export CC=clang
+export CXX=clang++
+
+tar -xf sqlite-autoconf-3250300.tar.gz
+pushd sqlite-autoconf-3250300
+
+CFLAGS="-fPIC" CPPFLAGS="-fPIC" ./configure --prefix /
+
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUIS} install DESTDIR=${ROOT}/out

cpython-macos/build-uuid.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export CC=clang
+export CXX=clang++
+
+tar -xf libuuid-${UUID_VERSION}.tar.gz
+pushd libuuid-${UUID_VERSION}
+
+CFLAGS="-fPIC" CPPFLAGS="-fPIC" ./configure --prefix=/
+
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out

cpython-macos/build-xz.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export CC=clang
+export CXX=clang++
+
+tar -xf xz-${XZ_VERSION}.tar.gz
+
+pushd xz-${XZ_VERSION}
+
+CFLAGS="-fPIC" CPPFLAGS="-fPIC" CCASFLAGS="-fPIC" ./configure \
+    --prefix=/ \
+    --disable-xz \
+    --disable-xzdec \
+    --disable-lzmadec \
+    --disable-lzmainfo \
+    --disable-lzma-links \
+    --disable-scripts
+
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out

cpython-macos/build-zlib.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export CC=clang
+export CXX=clang++
+
+tar -xf zlib-${ZLIB_VERSION}.tar.gz
+
+pushd zlib-${ZLIB_VERSION}
+
+CFLAGS="-fPIC" ./configure --prefix=/
+make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out