Skip to content

[build] E: Build lxml C extensions as .so #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
esaurez wants to merge 1 commit into from
Closed

[build] E: Build lxml C extensions as .so #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 58 additions & 5 deletions .nanvix/Makefile.nanvix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,22 @@
# MEMORY_SIZE=<size> NANVIX_HOME=<home> NANVIX_TOOLCHAIN=<toolchain> [target]
#
# Targets:
# all Build liblxml_etree.a, liblxml_elementpath.a, and the functional test ELF
# all Build liblxml_etree.{a,so}, liblxml_elementpath.{a,so}, and the functional test ELF
# test Run the functional test ELF on nanvixd.elf
# package Create a release tarball
# verify-package Verify the contents of the release tarball
# clean Remove build artifacts
#
# Dependencies:
# The shared-library build requires:
# * The Nanvix dynamic loader to honour `.init_array` constructors and
# DT_NEEDED resolution chains (esaurez/nanvix#27).
# * `libxml2.so`, `libxslt.so`, and `libexslt.so` to be present in the
# buildroot, shipped by the companion releases of esaurez/libxml2 and
# esaurez/libxslt.
# liblxml_etree.so is linked with DT_NEEDED libxslt.so libexslt.so
# libxml2.so so the loader pulls the lower-level libraries
# automatically at dlopen time, eliminating per-module duplication.

# ===========================================================================
# Global Variables
Expand All @@ -30,10 +41,12 @@ EXE = .elf
SRC_DIR := $(CURDIR)
STATICLIB_ETREE := dist/obj/liblxml_etree.a
STATICLIB_ELEMENTPATH := dist/obj/liblxml_elementpath.a
SHAREDLIB_ETREE := dist/obj/liblxml_etree.so
SHAREDLIB_ELEMENTPATH := dist/obj/liblxml_elementpath.so
TEST_SRC := .nanvix/test/test_lxml.c
TEST_ELF := test_lxml$(EXE)

_NANVIX_DOCKER_BUILD_GOALS := all $(STATICLIB_ETREE) $(STATICLIB_ELEMENTPATH) $(TEST_ELF)
_NANVIX_DOCKER_BUILD_GOALS := all $(STATICLIB_ETREE) $(STATICLIB_ELEMENTPATH) $(SHAREDLIB_ETREE) $(SHAREDLIB_ELEMENTPATH) $(TEST_ELF)
_NANVIX_GOALS := $(or $(MAKECMDGOALS),$(.DEFAULT_GOAL))

# Ensure required variables are defined.
Expand Down Expand Up @@ -68,21 +81,25 @@ endif
# Build Targets
# ===========================================================================

all: $(STATICLIB_ETREE) $(STATICLIB_ELEMENTPATH) $(TEST_ELF)
all: $(STATICLIB_ETREE) $(STATICLIB_ELEMENTPATH) $(SHAREDLIB_ETREE) $(SHAREDLIB_ELEMENTPATH) $(TEST_ELF)

# Compile the Cython-generated C sources with -fPIC so they can be
# linked into both the static .a (preserved for downstream consumers
# that still bundle lxml statically) and the new position-independent
# .so files below.
$(STATICLIB_ETREE) $(STATICLIB_ELEMENTPATH):
sh -c '\
export PATH="$(NANVIX_TOOLCHAIN)/bin:$$PATH" && \
mkdir -p dist/obj && \
i686-nanvix-gcc -m32 -march=pentiumpro -Os -fdata-sections -ffunction-sections \
i686-nanvix-gcc -m32 -march=pentiumpro -Os -fPIC -fdata-sections -ffunction-sections \
-I$(SRC_DIR)/.nanvix/nanvix-port/cpython-headers \
-I$(BUILDROOT_PATH)/include \
-I$(BUILDROOT_PATH)/include/libxml2 \
-I$(SYSROOT_PATH)/include \
-I$(SRC_DIR)/src \
-I$(SRC_DIR)/src/lxml/includes \
-c $(SRC_DIR)/src/lxml/etree.c -o dist/obj/lxml_etree.o && \
i686-nanvix-gcc -m32 -march=pentiumpro -Os -fdata-sections -ffunction-sections \
i686-nanvix-gcc -m32 -march=pentiumpro -Os -fPIC -fdata-sections -ffunction-sections \
-I$(SRC_DIR)/.nanvix/nanvix-port/cpython-headers \
-I$(BUILDROOT_PATH)/include \
-I$(BUILDROOT_PATH)/include/libxml2 \
Expand All @@ -95,6 +112,36 @@ $(STATICLIB_ETREE) $(STATICLIB_ELEMENTPATH):
i686-nanvix-ranlib dist/obj/liblxml_etree.a && \
i686-nanvix-ranlib dist/obj/liblxml_elementpath.a'

# liblxml_etree.so: links against libxslt.so + libxml2.so (DT_NEEDED
# entries), with only the cython-generated lxml.etree.c embedded.
# libxslt, libxml2, and libz live in their own .so files and are
# pulled in transitively by the loader at dlopen time, eliminating
# the multi-megabyte duplication that a self-contained build would
# cause. CPython C API symbols (PyType_Type, PyExc_*, ...) and
# libc/libm/libposix entry points stay unresolved and bind against
# python.elf's .dynsym at dlopen time.
$(SHAREDLIB_ETREE): $(STATICLIB_ETREE)
sh -c '\
export PATH="$(NANVIX_TOOLCHAIN)/bin:$$PATH" && \
i686-nanvix-gcc -shared -fPIC -nostdlib \
-Wl,-soname,liblxml_etree.so -Wl,-z,noexecstack \
-L$(BUILDROOT_PATH)/lib \
-Wl,--whole-archive $(STATICLIB_ETREE) -Wl,--no-whole-archive \
-lxslt -lexslt -lxml2 \
-o $(SHAREDLIB_ETREE)'

# liblxml_elementpath.so: pure-Cython module with no native deps
# beyond CPython itself, so no DT_NEEDED entries.
$(SHAREDLIB_ELEMENTPATH): $(STATICLIB_ELEMENTPATH)
sh -c '\
export PATH="$(NANVIX_TOOLCHAIN)/bin:$$PATH" && \
i686-nanvix-gcc -shared -fPIC -nostdlib \
-Wl,-soname,liblxml_elementpath.so -Wl,-z,noexecstack \
-Wl,--whole-archive \
$(STATICLIB_ELEMENTPATH) \
-Wl,--no-whole-archive \
-o $(SHAREDLIB_ELEMENTPATH)'

$(TEST_ELF): $(TEST_SRC)
@test -f $(STATICLIB_ETREE) || { echo " FAIL: $(STATICLIB_ETREE) not found; run 'build' first"; exit 1; }
@test -f $(STATICLIB_ELEMENTPATH) || { echo " FAIL: $(STATICLIB_ELEMENTPATH) not found; run 'build' first"; exit 1; }
Expand Down Expand Up @@ -138,13 +185,17 @@ package:
$(eval ARTIFACT_NAME := lxml-$(PLATFORM)-$(PROCESS_MODE)-$(MEMORY_SIZE))
@test -f $(STATICLIB_ETREE) || { echo " FAIL: $(STATICLIB_ETREE) not found; run 'build' first"; exit 1; }
@test -f $(STATICLIB_ELEMENTPATH) || { echo " FAIL: $(STATICLIB_ELEMENTPATH) not found; run 'build' first"; exit 1; }
@test -f $(SHAREDLIB_ETREE) || { echo " FAIL: $(SHAREDLIB_ETREE) not found; run 'build' first"; exit 1; }
@test -f $(SHAREDLIB_ELEMENTPATH) || { echo " FAIL: $(SHAREDLIB_ELEMENTPATH) not found; run 'build' first"; exit 1; }
rm -rf dist/$(ARTIFACT_NAME)
mkdir -p dist/$(ARTIFACT_NAME)/sysroot/lib \
dist/$(ARTIFACT_NAME)/sysroot/python-packages/lxml/html \
dist/$(ARTIFACT_NAME)/sysroot/python-packages/lxml/isoschematron \
dist/$(ARTIFACT_NAME)/sysroot/python-packages/lxml/includes
cp -f $(STATICLIB_ETREE) dist/$(ARTIFACT_NAME)/sysroot/lib/
cp -f $(STATICLIB_ELEMENTPATH) dist/$(ARTIFACT_NAME)/sysroot/lib/
cp -f $(SHAREDLIB_ETREE) dist/$(ARTIFACT_NAME)/sysroot/lib/
cp -f $(SHAREDLIB_ELEMENTPATH) dist/$(ARTIFACT_NAME)/sysroot/lib/
cp -f $(SRC_DIR)/src/lxml/*.py dist/$(ARTIFACT_NAME)/sysroot/python-packages/lxml/
-cp -f $(SRC_DIR)/src/lxml/html/*.py dist/$(ARTIFACT_NAME)/sysroot/python-packages/lxml/html/ 2>/dev/null || true
-cp -f $(SRC_DIR)/src/lxml/isoschematron/*.py dist/$(ARTIFACT_NAME)/sysroot/python-packages/lxml/isoschematron/ 2>/dev/null || true
Expand All @@ -158,6 +209,8 @@ verify-package:
@test -f "dist/$(ARTIFACT_NAME).tar.gz" || { echo " FAIL: tarball not found"; exit 1; }
@tar tzf "dist/$(ARTIFACT_NAME).tar.gz" | grep -q 'sysroot/lib/liblxml_etree.a' || { echo " FAIL: missing liblxml_etree.a"; exit 1; }
@tar tzf "dist/$(ARTIFACT_NAME).tar.gz" | grep -q 'sysroot/lib/liblxml_elementpath.a' || { echo " FAIL: missing liblxml_elementpath.a"; exit 1; }
@tar tzf "dist/$(ARTIFACT_NAME).tar.gz" | grep -q 'sysroot/lib/liblxml_etree.so' || { echo " FAIL: missing liblxml_etree.so"; exit 1; }
@tar tzf "dist/$(ARTIFACT_NAME).tar.gz" | grep -q 'sysroot/lib/liblxml_elementpath.so' || { echo " FAIL: missing liblxml_elementpath.so"; exit 1; }
@tar tzf "dist/$(ARTIFACT_NAME).tar.gz" | grep -q 'sysroot/python-packages/lxml/' || { echo " FAIL: missing python-packages"; exit 1; }
@echo " PASS: lxml package verification"

Expand Down
2 changes: 2 additions & 0 deletions .nanvix/z.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ def docker_config(self, image: str):
cfg.output_files = [
"dist/obj/liblxml_etree.a",
"dist/obj/liblxml_elementpath.a",
"dist/obj/liblxml_etree.so",
"dist/obj/liblxml_elementpath.so",
"test_lxml.elf",
]
return cfg
Expand Down