pgcrypto

build-pglite.sh

@@ -7,13 +7,15 @@
 # final output folder
 INSTALL_FOLDER=${INSTALL_FOLDER:-"/install/pglite"}
 
+PGLITE_CFLAGS="-D__PGLITE__"
 # build with optimizations by default aka release
-PGLITE_CFLAGS="-O2"
 if [ "$DEBUG" = true ]
 then
     echo "pglite: building debug version."
-    PGLITE_CFLAGS="-g -gsource-map --no-wasm-opt"
+    # -sDYLINK_DEBUG=2
+    PGLITE_CFLAGS="$PGLITE_CFLAGS -g -gsource-map --no-wasm-opt"
 else
+    PGLITE_CFLAGS="$PGLITE_CFLAGS -O2"
     echo "pglite: building release version."
     # we shouldn't need to do this, but there's a bug somewhere that prevents a successful build if this is set
     unset DEBUG
@@ -50,12 +52,31 @@ PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
 -sTOTAL_MEMORY=32MB \
 --embed-file $(pwd)/other/PGPASSFILE@/home/web_user/.pgpass"
 
+CONFIGURE_PARAMS="\
+ac_cv_exeext=.js \
+--host aarch64-unknown-linux-gnu \
+--disable-spinlocks \
+--disable-largefile \
+--without-llvm \
+--without-pam \
+--with-openssl=no \
+--without-readline \
+--without-icu \
+--with-uuid=ossp \
+--with-zlib \
+--with-libxml \
+--with-libxslt \
+--with-template=emscripten \
+--with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2:$(pwd)/pglite/includes \
+--with-libraries=$INSTALL_PREFIX/lib \
+--prefix=$INSTALL_FOLDER"
+
 # Step 1: configure the project
 if [ "$RUN_CONFIGURE" = true ]; then
     LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" \
     LDFLAGS_SL="-sSIDE_MODULE=1" \
     LDFLAGS_EX=$PGLITE_EMSCRIPTEN_FLAGS \
-    CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.js --host aarch64-unknown-linux-gnu --disable-spinlocks --disable-largefile --without-llvm  --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2:$(pwd)/pglite/includes --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; }
+    CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure $CONFIGURE_PARAMS || { echo 'error: emconfigure failed' ; exit 11; }
 else
     echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}"
 fi
@@ -65,11 +86,17 @@ emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscri
 emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; }
 
 # Step 3.1: make all contrib extensions - do not install
+# Step 3.1.1 pgcrypto - special case
+cd ./pglite/ && ./build-pgcrypto.sh && cd ../
+# Step 3.1.2 all the rest of contrib
 emmake make PORTNAME=emscripten -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; }
 # Step 3.2: make dist contrib extensions - this will create an archive for each extension
 emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; }
 # the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive
 
+# hack for pgcrypto. since we're linking lssl and lcrypto directly to the extension, their respective symbols should not be exported
+find / -name pgcrypto.imports -exec rm -f {} \;
+
 # Step 4: make and dist other extensions
 SAVE_PATH=$PATH
 PATH=$PATH:$INSTALL_FOLDER/bin
@@ -87,5 +114,20 @@ PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
 -sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \
 -sERROR_ON_UNDEFINED_SYMBOLS=0 \
 -sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS"
+
+PGROOT=/install/pglite
+PGLITE_PRELOAD="\
+--preload-file ${PGROOT}/share/postgresql@/tmp/pglite/share/postgresql \
+--preload-file ${PGROOT}/lib/postgresql@/tmp/pglite/lib/postgresql \
+--preload-file $(pwd)/other/password@/tmp/pglite/password \
+--preload-file $(pwd)/other/PGPASSFILE@/home/web_user/.pgpass \
+--preload-file $(pwd)/other/empty@/tmp/pglite/bin/postgres \
+--preload-file $(pwd)/other/empty@/tmp/pglite/bin/initdb"
+
+PGLITE_EMSCRIPTEN_LIBS="-lnodefs.js -lidbfs.js"
+EXPORTED_FUNCTIONS="-sEXPORTED_FUNCTIONS=@/tmp/exported_functions.txt"
+
+# -sDYLINK_DEBUG=2 use this for debugging missing exported symbols (ex when an extension calls a pgcore function that hasn't been exported)
+
 # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available.
-PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }
+PGLITE_FLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS $PGLITE_PRELOAD $PGLITE_EMSCRIPTEN_LIBS $EXPORTED_FUNCTIONS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }

build-with-docker.sh

@@ -3,11 +3,12 @@
 DOCKER_WORKSPACE=$(pwd)
 
 docker run $@ \
+  --name pglite-builder \
   --rm \
   -e DEBUG=${DEBUG:-false} \
   --workdir=${DOCKER_WORKSPACE} \
   -v .:${DOCKER_WORKSPACE}:rw \
   -v ./dist:/install/pglite:rw \
-  electricsql/pglite-builder:3.1.74_2 \
+  electricsql/pglite-builder:3.1.74_4 \
   ./build-pglite.sh
 

contrib/Makefile

@@ -57,6 +57,10 @@ else
 ALWAYS_SUBDIRS += pgcrypto sslinfo
 endif
 
+ifeq ($(PORTNAME), emscripten)
+SUBDIRS += pgcrypto
+endif
+
 ifneq ($(with_uuid),no)
 SUBDIRS += uuid-ossp
 else

contrib/pgcrypto/Makefile

@@ -62,9 +62,6 @@ endif
 # shared library link.  (The order in which you list them here doesn't
 # matter.)
 SHLIB_LINK += $(filter -lcrypto -lz, $(LIBS))
-ifeq ($(PORTNAME), emscripten)
-SHLIB_LINK += -lcrypto
-endif
 ifeq ($(PORTNAME), win32)
 SHLIB_LINK += $(filter -leay32, $(LIBS))
 # those must be at the end

pglite-wasm/builder/Dockerfile

@@ -22,7 +22,7 @@ WORKDIR /install/libs
 WORKDIR /src
 
 # ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn"
-ENV EMCC_COMMON_FLAGS="-O2 -fPIC"
+ENV EMCC_COMMON_FLAGS="-O2 -fPIC -sWASM_BIGINT"
 
 WORKDIR /src
 RUN curl -L https://www.zlib.net/zlib-1.3.1.tar.gz | tar -xz
@@ -34,7 +34,7 @@ RUN ${LLVM_NM} /install/libs/lib/libz.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed
 WORKDIR /src
 RUN curl -L https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.14.5/libxml2-v2.14.5.tar.gz | tar -xz
 WORKDIR /src/libxml2-v2.14.5
-RUN ./autogen.sh --with-python=no
+RUN ./autogen.sh --with-python=no --with-threads=no
 RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs
 RUN emmake make -j && emmake make install
 # extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS 
@@ -52,11 +52,12 @@ RUN ${LLVM_NM} /install/libs/lib/libxslt.a | awk '$2 ~ /^[TDB]$/ {print $3}' | s
 WORKDIR /src
 RUN curl -L https://github.com/openssl/openssl/releases/download/openssl-3.0.17/openssl-3.0.17.tar.gz | tar xz
 WORKDIR /src/openssl-3.0.17
-RUN emconfigure ./Configure no-tests linux-generic64 --prefix=/install/libs
+RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./config no-sse2 no-hw no-asm no-tests no-threads no-shared linux-generic32 --prefix=/install/libs
 RUN sed -i 's|^CROSS_COMPILE.*$|CROSS_COMPILE=|g' Makefile # see https://github.com/emscripten-core/emscripten/issues/19597#issue-1754476454
-RUN emmake make -j && emmake make install
+RUN emmake make -j && emmake make install_sw install_ssldirs
 # extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS 
 RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libssl.exports
+RUN ${LLVM_NM} /install/libs/lib/libcrypto.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libcrypto.exports
 
 WORKDIR /src
 RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz

pglite-wasm/included.pglite.imports

@@ -1,33 +1,179 @@
+__errno_location
+abort
+accept
+appendStringInfo
+appendStringInfoChar
+appendStringInfoString
+atexit
+atoi
+bind
+BuildTupleFromCStrings
+calloc
+chmod
+clearerr
+clock_gettime
 close
+closedir
+closelog
+connect
+cstring_to_text
+cstring_to_text_with_len
+deconstruct_array_builtin
+dladdr
+dlclose
+dlerror
+dlopen
+dlsym
+downcase_truncate_identifier
+end_MultiFuncCall
+enlargeStringInfo
+errcode
+errfinish
+errmsg
+errmsg_internal
+errstart
+errstart_cold
+fclose
 fcntl
+fdopen
+feof
+ferror
+fflush
+fgets
+fileno
+fiprintf
+fopen
+fputc
+fputs
+fread
 free
+freeaddrinfo
+fseek
+fstat
+ftell
+fwrite
+gai_strerror
+gen_random_uuid
+get_call_result_type
+getaddrinfo
+GetDatabaseEncoding
+getegid
+getenv
+geteuid
+getgid
+gethostbyname
+getnameinfo
 getpid
+getsockname
+getsockopt
 gettimeofday
+getuid
 gmtime
+HeapTupleHeaderGetDatum
+init_MultiFuncCall
+initStringInfo
 interactive_one
 ioctl
 isalnum
 isxdigit
+listen
 lowerstr
+lseek
+madvise
 main
 malloc
+memchr
 memcmp
 memcpy
+memmove
+MemoryContextAlloc
+MemoryContextAllocZero
 memset
+mlock
+mmap
+mprotect
+munmap
 nanosleep
+ntohs
 open
+opendir
+openlog
+palloc
+palloc0
+per_MultiFuncCall
+perror
+pfree
+pg_any_to_server
+pg_detoast_datum
+pg_detoast_datum_packed
+pg_do_encoding_conversion
+pg_is_ascii
+pg_strcasecmp
+pg_strong_random
+pg_vsnprintf
 pgl_backend
 pgl_initdb
 pgl_shutdown
+ProcessInterrupts
+pstrdup
+puts
+qsort
 rand
 read
+readdir
 readstoplist
 realloc
+recvfrom
+repalloc
+ResourceOwnerEnlarge
+ResourceOwnerForget
+ResourceOwnerRemember
 searchstoplist
+select
+sendto
 set_read_write_cbs
+setbuf
+setsockopt
+shutdown
+sigaction
+signal
+siprintf
+sleep
+snprintf
 socket
 srand
+sscanf
+stat
+stderr
+strcat
+strchr
 strcmp
+strcpy
+strcspn
+strdup
+strerror
+strerror_r
 strftime
+strlcpy
 strlen
-strtoul
+strncat
+strncmp
+strncpy
+strrchr
+strspn
+strstr
+strtol
+strtoul
+sysconf
+syslog
+tcgetattr
+tcsetattr
+text_to_cstring
+text_to_cstring_buffer
+time
+tolower
+TupleDescGetAttInMetadata
+usleep
+vfprintf
+vsnprintf
+write

pglite-wasm/interactive_one.c

@@ -304,7 +304,7 @@ interactive_one(int packetlen, int peek) {
     }
 
 incoming:
-#if defined(__EMSCRIPTEN__) || defined(__wasi__) //PGDEBUG
+#if defined(__PGLITE__) //PGDEBUG
 #   include "pgl_sjlj.c"
 #else
     #error "sigsetjmp unsupported"