From 717c09b5052858d97cdfc68582a29c7170f2710e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 9 Jan 2022 01:37:36 -0800 Subject: [PATCH 1/9] bpo-40280: Add build target for Emscripten/node.js This adds a "python.js" target to the Makefile. We enable the raw node fs mode as suggested by @thomasballinger as it will make running the test suite under node easier, c.f. https://github.com/ethanhs/python-wasm/issues/36. The wasm_asset.py script is updated to include the test module. Care is taken to skip compiling the Python files that are examples of bad syntax or encoding. This also turns on memory growth as early into the test suite I was running into OOMs. --- Makefile.pre.in | 10 +++++++++- Tools/wasm/wasm_assets.py | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index fbd4c3a23fd81b..efa047b7e5badc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -833,6 +833,7 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) # wasm32-emscripten build # wasm assets directory is relative to current build dir, e.g. "./usr/local". # --preload-file turns a relative asset path into an absolute path. +# With node.js, we use the rawfs which is much easier for tests WASM_ASSETS_DIR=".$(prefix)" WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py" @@ -844,7 +845,14 @@ $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ - -s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR) + -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ + --preload-file $(WASM_ASSETS_DIR) + +python.js: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ + $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ + -s ASSERTIONS=1 -s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 \ + -s EXIT_RUNTIME=1 ########################################################################## # Build static libmpdec.a diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py index 6a4027184030f7..3e2c6f42cca86e 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/wasm_assets.py @@ -100,6 +100,21 @@ "unittest/test/", ) +SKIP_COMPILE = ( + "test/bad_coding.py", + "test/bad_coding2.py", + "test/badsyntax_3131.py", + "test/badsyntax_future3.py", + "test/badsyntax_future4.py", + "test/badsyntax_future5.py", + "test/badsyntax_future6.py", + "test/badsyntax_future7.py", + "test/badsyntax_future8.py", + "test/badsyntax_future9.py", + "test/badsyntax_future10.py", + "test/badsyntax_pep3120.py", +) + OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES} OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS) @@ -124,7 +139,9 @@ def create_stdlib_zip( continue if entry in OMIT_ABSOLUTE: continue - if entry.name.endswith(".py") or entry.is_dir(): + if entry in SKIP_COMPILE: + pzf.write(entry) + elif entry.name.endswith(".py") or entry.is_dir(): # writepy() writes .pyc files (bytecode). pzf.writepy(entry, filterfunc=filterfunc) for entry in sysconfig_data: From a862bcefc584ff87945b24c7c8bd1edf0ec23e14 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 10 Jan 2022 01:58:48 -0800 Subject: [PATCH 2/9] Update Makefile.pre.in Co-authored-by: Christian Heimes --- Makefile.pre.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index efa047b7e5badc..df7011417beeba 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -848,11 +848,11 @@ python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ --preload-file $(WASM_ASSETS_DIR) -python.js: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) +python.js: Programs/python.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ - -s ASSERTIONS=1 -s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 \ - -s EXIT_RUNTIME=1 + -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ + -s NODERAWFS=1 -s EXIT_RUNTIME=1 ########################################################################## # Build static libmpdec.a From 6d88ce6786678b32cbfbe9affcc62f49563ea692 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 10 Jan 2022 09:29:24 -0800 Subject: [PATCH 3/9] Change target name to python-rawfs.js --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index df7011417beeba..d8070825a0cb77 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -848,7 +848,7 @@ python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ --preload-file $(WASM_ASSETS_DIR) -python.js: Programs/python.o $(LIBRARY_DEPS) +python-rawfs.js: Programs/python.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ From 3ee56876c5a7aca94c603f0536fdc746deec2e0d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 10 Jan 2022 09:30:34 -0800 Subject: [PATCH 4/9] Remove special casing for test files --- Tools/wasm/wasm_assets.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py index 3e2c6f42cca86e..6a4027184030f7 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/wasm_assets.py @@ -100,21 +100,6 @@ "unittest/test/", ) -SKIP_COMPILE = ( - "test/bad_coding.py", - "test/bad_coding2.py", - "test/badsyntax_3131.py", - "test/badsyntax_future3.py", - "test/badsyntax_future4.py", - "test/badsyntax_future5.py", - "test/badsyntax_future6.py", - "test/badsyntax_future7.py", - "test/badsyntax_future8.py", - "test/badsyntax_future9.py", - "test/badsyntax_future10.py", - "test/badsyntax_pep3120.py", -) - OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES} OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS) @@ -139,9 +124,7 @@ def create_stdlib_zip( continue if entry in OMIT_ABSOLUTE: continue - if entry in SKIP_COMPILE: - pzf.write(entry) - elif entry.name.endswith(".py") or entry.is_dir(): + if entry.name.endswith(".py") or entry.is_dir(): # writepy() writes .pyc files (bytecode). pzf.writepy(entry, filterfunc=filterfunc) for entry in sysconfig_data: From ee99490f9c1e66ae4071935ca3264db784aa4d0e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 10 Jan 2022 20:15:28 -0800 Subject: [PATCH 5/9] Fix threading for nodejs --- Makefile.pre.in | 13 +++++--- Modules/socketmodule.c | 3 +- Tools/wasm/config.site-wasm32-emscripten | 7 +++++ configure | 40 +++++++++++++++++++----- configure.ac | 34 +++++++++++++++----- 5 files changed, 75 insertions(+), 22 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index d8070825a0cb77..a985679a9d63b7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -38,6 +38,7 @@ CC= @CC@ CXX= @CXX@ MAINCC= @MAINCC@ LINKCC= @LINKCC@ +LINKCC_BUILDPYTHON = @LINKCC_BUILDPYTHON@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ @@ -317,6 +318,7 @@ COVERAGE_REPORT_OPTIONS=--no-branch-coverage --title "CPython lcov report" ########################################################################## + # Modules MODULE_OBJS= \ Modules/config.o \ @@ -717,7 +719,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c # Build the interpreter $(BUILDPYTHON): Programs/python.o $(LIBRARY_DEPS) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) + $(LINKCC_BUILDPYTHON) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -845,14 +847,15 @@ $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ - -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ - --preload-file $(WASM_ASSETS_DIR) + -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 -s USE_PTHREADS \ + -s PROXY_TO_PTHREAD --preload-file $(WASM_ASSETS_DIR) -python-rawfs.js: Programs/python.o $(LIBRARY_DEPS) +python-rawfs.js: Programs/python.o $(LIBRARY_DEPS) pybuilddir.txt $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ - -s NODERAWFS=1 -s EXIT_RUNTIME=1 + -s NODERAWFS=1 -s EXIT_RUNTIME=1 -s USE_PTHREADS \ + -s PROXY_TO_PTHREAD ########################################################################## # Build static libmpdec.a diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 89e93c58187c96..dbd7d4dc63ea7b 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -5114,6 +5114,7 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds) &family, &type, &proto, &fdobj)) return -1; + #ifdef MS_WINDOWS /* In this case, we don't use the family, type and proto args */ if (fdobj == NULL || fdobj == Py_None) @@ -7926,7 +7927,7 @@ PyInit__socket(void) #ifdef IPPROTO_VRRP PyModule_AddIntMacro(m, IPPROTO_VRRP); #endif -#ifdef IPPROTO_SCTP +#if defined(IPPROTO_SCTP) && !defined(__EMSCRIPTEN__) PyModule_AddIntMacro(m, IPPROTO_SCTP); #endif #ifdef IPPROTO_BIP diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/config.site-wasm32-emscripten index b291c802e1e4dd..143d0fb1d5881d 100644 --- a/Tools/wasm/config.site-wasm32-emscripten +++ b/Tools/wasm/config.site-wasm32-emscripten @@ -7,6 +7,13 @@ # Partly based on pyodide's pyconfig.undefs.h file. # +ac_exeext=.wasm + +# Emscripten will build code that includes pthreads without the flag, but fail to run... +ac_cv_pthread_is_default=no +ac_cv_kthread=no +ac_cv_pthread=yes + # cannot be detected in cross builds ac_cv_buggy_getaddrinfo=no diff --git a/configure b/configure index 9712446d24c11d..88b89a34fbc722 100755 --- a/configure +++ b/configure @@ -878,6 +878,7 @@ AR GNULD EXPORTSFROM EXPORTSYMS +LINKCC_BUILDPYTHON LINKCC LDVERSION RUNSHARED @@ -6326,6 +6327,25 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LINKCC" >&5 $as_echo "$LINKCC" >&6; } + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking LINKCC_BUILDPYTHON" >&5 +$as_echo_n "checking LINKCC_BUILDPYTHON... " >&6; } +if test -z "$LINKCC_BUILDPYTHON" +then + # the .wasm executable doesn't support building with pthreads, + # so we hack around the symbol errors as they will be provided by + # Emscripten + case $ac_sys_system in #( + Emscripten) : + LINKCC_BUILDPYTHON="emcc -s ERROR_ON_UNDEFINED_SYMBOLS=0" ;; #( + *) : + LINKCC_BUILDPYTHON='$(LINKCC)' + ;; +esac +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $LINKCC_BUILDPYTHON" >&5 +$as_echo "$LINKCC_BUILDPYTHON" >&6; } + # EXPORTSYMS holds the list of exported symbols for AIX. # EXPORTSFROM holds the module name exporting symbols on AIX. EXPORTSYMS= @@ -8276,7 +8296,6 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread_is_default" >&5 $as_echo "$ac_cv_pthread_is_default" >&6; } - if test $ac_cv_pthread_is_default = yes then ac_cv_kpthread=no @@ -8451,16 +8470,21 @@ fi if test $ac_cv_cxx_thread = yes then - echo 'void foo();int main(){foo();}void foo(){}' > conftest.$ac_ext - $CXX -c conftest.$ac_ext 2>&5 - if $CXX -o conftest$ac_exeext conftest.$ac_objext 2>&5 \ - && test -s conftest$ac_exeext && ./conftest$ac_exeext - then + if test "$cross_compiling" = yes; then + # assume that it works ac_cv_cxx_thread=yes else - ac_cv_cxx_thread=no + echo 'void foo();int main(){foo();}void foo(){}' > conftest.$ac_ext + $CXX -c conftest.$ac_ext 2>&5 + if $CXX -o conftest$ac_exeext conftest.$ac_objext 2>&5 \ + && test -s conftest$ac_exeext && ./conftest$ac_exeext + then + ac_cv_cxx_thread=yes + else + ac_cv_cxx_thread=no + fi + rm -fr conftest* fi - rm -fr conftest* fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_thread" >&5 $as_echo "$ac_cv_cxx_thread" >&6; } diff --git a/configure.ac b/configure.ac index 1720b9bfbee379..3e10c3edb36a26 100644 --- a/configure.ac +++ b/configure.ac @@ -1162,6 +1162,20 @@ then fi AC_MSG_RESULT($LINKCC) +AC_SUBST(LINKCC_BUILDPYTHON) +AC_MSG_CHECKING(LINKCC_BUILDPYTHON) +if test -z "$LINKCC_BUILDPYTHON" +then + # the .wasm executable doesn't support building with pthreads, + # so we hack around the symbol errors as they will be provided by + # Emscripten + AS_CASE([$ac_sys_system], + [Emscripten], [LINKCC_BUILDPYTHON="emcc -s ERROR_ON_UNDEFINED_SYMBOLS=0"], + [LINKCC_BUILDPYTHON='$(LINKCC)'] + ) +fi +AC_MSG_RESULT($LINKCC_BUILDPYTHON) + # EXPORTSYMS holds the list of exported symbols for AIX. # EXPORTSFROM holds the module name exporting symbols on AIX. EXPORTSYMS= @@ -2125,7 +2139,6 @@ int main(){ ],[ac_cv_pthread_is_default=no],[ac_cv_pthread_is_default=no]) ]) - if test $ac_cv_pthread_is_default = yes then ac_cv_kpthread=no @@ -2233,16 +2246,21 @@ fi if test $ac_cv_cxx_thread = yes then - echo 'void foo();int main(){foo();}void foo(){}' > conftest.$ac_ext - $CXX -c conftest.$ac_ext 2>&5 - if $CXX -o conftest$ac_exeext conftest.$ac_objext 2>&5 \ - && test -s conftest$ac_exeext && ./conftest$ac_exeext - then + if test "$cross_compiling" = yes; then + # assume that it works ac_cv_cxx_thread=yes else - ac_cv_cxx_thread=no + echo 'void foo();int main(){foo();}void foo(){}' > conftest.$ac_ext + $CXX -c conftest.$ac_ext 2>&5 + if $CXX -o conftest$ac_exeext conftest.$ac_objext 2>&5 \ + && test -s conftest$ac_exeext && ./conftest$ac_exeext + then + ac_cv_cxx_thread=yes + else + ac_cv_cxx_thread=no + fi + rm -fr conftest* fi - rm -fr conftest* fi AC_MSG_RESULT($ac_cv_cxx_thread) fi From 3a0dd1ce30b0f6b03bebc8fcd6a1b2061accfca8 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 11 Jan 2022 09:15:41 -0800 Subject: [PATCH 6/9] Update Makefile.pre.in Co-authored-by: Christian Heimes --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a985679a9d63b7..1384dd56c6686b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -848,7 +848,7 @@ python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 -s USE_PTHREADS \ - -s PROXY_TO_PTHREAD --preload-file $(WASM_ASSETS_DIR) + -s PROXY_TO_PTHREAD --preload-file $(WASM_ASSETS_DIR) python-rawfs.js: Programs/python.o $(LIBRARY_DEPS) pybuilddir.txt $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ From faf52a4c9560a6b2ed02566d760d94e723cc870d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 11 Jan 2022 09:15:47 -0800 Subject: [PATCH 7/9] Update Makefile.pre.in Co-authored-by: Christian Heimes --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 1384dd56c6686b..7e5582286bf8fe 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -855,7 +855,7 @@ python-rawfs.js: Programs/python.o $(LIBRARY_DEPS) pybuilddir.txt $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 \ -s NODERAWFS=1 -s EXIT_RUNTIME=1 -s USE_PTHREADS \ - -s PROXY_TO_PTHREAD + -s PROXY_TO_PTHREAD ########################################################################## # Build static libmpdec.a From bdcaa48382ecbdf69cd15ada4df61299424b2237 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 11 Jan 2022 09:24:39 -0800 Subject: [PATCH 8/9] Update Makefile.pre.in Co-authored-by: Christian Heimes --- Makefile.pre.in | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 7e5582286bf8fe..08ca9bb7cfe3f8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -318,7 +318,6 @@ COVERAGE_REPORT_OPTIONS=--no-branch-coverage --title "CPython lcov report" ########################################################################## - # Modules MODULE_OBJS= \ Modules/config.o \ From 528426549019b2559de3cdb0382ce26efbc8d4b4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 11 Jan 2022 09:24:44 -0800 Subject: [PATCH 9/9] Update Modules/socketmodule.c Co-authored-by: Christian Heimes --- Modules/socketmodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index dbd7d4dc63ea7b..050bd4181c623a 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -5114,7 +5114,6 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds) &family, &type, &proto, &fdobj)) return -1; - #ifdef MS_WINDOWS /* In this case, we don't use the family, type and proto args */ if (fdobj == NULL || fdobj == Py_None)