From a1fe8c48aae5058bbcd18c03194858700c249ef5 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Thu, 16 Oct 2014 08:46:32 +0300 Subject: [PATCH 01/10] python3: reduce bloat by removing *.pyo and *.pyc files Seems that if you add a package folder this would also include the compiled python3 files which increases fw size. Signed-off-by: Alexandru Ardelean --- lang/python3/files/python3-package.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/lang/python3/files/python3-package.mk b/lang/python3/files/python3-package.mk index e85b64819..090d84601 100644 --- a/lang/python3/files/python3-package.mk +++ b/lang/python3/files/python3-package.mk @@ -32,6 +32,7 @@ define Py3Package $(call shexport,Py3Package/$(1)/filespec) define Package/$(1)/install + find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" | xargs rm -f @$(SH_FUNC) getvar $$(call shvar,Py3Package/$(1)/filespec) | ( \ IFS='|'; \ while read fop fspec fperm; do \ From 885fe99cd29ea6b056e816ff78dd750b0e7dde8b Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Thu, 16 Oct 2014 16:15:56 +0300 Subject: [PATCH 02/10] python3: enforce the existence of files in the filespecs Seems that this allows some goofs, because some files silently do not get copied and the build succeeds, even though it shouldn't. Signed-off-by: Alexandru Ardelean --- lang/python3/files/python3-package.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lang/python3/files/python3-package.mk b/lang/python3/files/python3-package.mk index 090d84601..7808d9ecd 100644 --- a/lang/python3/files/python3-package.mk +++ b/lang/python3/files/python3-package.mk @@ -37,6 +37,10 @@ define Py3Package IFS='|'; \ while read fop fspec fperm; do \ if [ "$$$$$$$$fop" = "+" ]; then \ + if [ ! -e "$(PKG_INSTALL_DIR)$$$$$$$$fspec" ]; then \ + echo "File not found '$(PKG_INSTALL_DIR)$$$$$$$$fspec'"; \ + exit 1; \ + fi; \ dpath=`dirname "$$$$$$$$fspec"`; \ if [ -n "$$$$$$$$fperm" ]; then \ dperm="-m$$$$$$$$fperm"; \ From 8c4da0367b50cf7b4fc8941ba7af3008d86b6daf Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Fri, 17 Oct 2014 16:34:12 +0300 Subject: [PATCH 03/10] python3: patch setup.py so that it does not include system include dirs Seems that the Python C extensions were being (or at least trying to be) build using '/usr/include' as the first include folder. Seems this issue was already fixed on MacOS X and now we've extended it for our case. Signed-off-by: Alexandru Ardelean --- ...not-add-include-dirs-when-cross-compiling.patch | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lang/python3/patches/120-do-not-add-include-dirs-when-cross-compiling.patch diff --git a/lang/python3/patches/120-do-not-add-include-dirs-when-cross-compiling.patch b/lang/python3/patches/120-do-not-add-include-dirs-when-cross-compiling.patch new file mode 100644 index 000000000..44be1c8ff --- /dev/null +++ b/lang/python3/patches/120-do-not-add-include-dirs-when-cross-compiling.patch @@ -0,0 +1,14 @@ +diff --git a/setup.py b/setup.py +index 93f390f..ace1494 100644 +--- a/setup.py ++++ b/setup.py +@@ -461,7 +461,8 @@ class PyBuildExt(build_ext): + add_dir_to_list(dir_list, directory) + + if os.path.normpath(sys.base_prefix) != '/usr' \ +- and not sysconfig.get_config_var('PYTHONFRAMEWORK'): ++ and not sysconfig.get_config_var('PYTHONFRAMEWORK') \ ++ and not cross_compiling: + # OSX note: Don't add LIBDIR and INCLUDEDIR to building a framework + # (PYTHONFRAMEWORK is set) to avoid # linking problems when + # building a framework with different architectures than From e0abcb569b6c676a869222ac3c6c0408bcd644f3 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Mon, 20 Oct 2014 18:09:57 +0300 Subject: [PATCH 04/10] python: add ac_cv_buggy_getaddrinfo=no to config.site file Required when IPv6 support gets enabled. Signed-off-by: Alexandru Ardelean --- lang/python3/files/config.site | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lang/python3/files/config.site b/lang/python3/files/config.site index f4113357a..cfa56b89b 100644 --- a/lang/python3/files/config.site +++ b/lang/python3/files/config.site @@ -8,11 +8,5 @@ ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no - -# FIXME: moved from Makefile here -#ac_cv_have_chflags=no \ -#ac_cv_have_lchflags=no \ -#ac_cv_py_format_size_t=no \ -#ac_cv_have_long_long_format=yes \ -#ac_cv_buggy_getaddrinfo=no \ +ac_cv_buggy_getaddrinfo=no From 5bf4866fd9f87a07bb3eaad167d7e9317564ed6b Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Fri, 17 Oct 2014 17:04:26 +0300 Subject: [PATCH 05/10] python3: add patch to disable package compiles/tests during cross-builts Signed-off-by: Alexandru Ardelean --- .../130-do-not-run-distutils-tests.patch | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 lang/python3/patches/130-do-not-run-distutils-tests.patch diff --git a/lang/python3/patches/130-do-not-run-distutils-tests.patch b/lang/python3/patches/130-do-not-run-distutils-tests.patch new file mode 100644 index 000000000..0291eb198 --- /dev/null +++ b/lang/python3/patches/130-do-not-run-distutils-tests.patch @@ -0,0 +1,37 @@ +diff --git a/Makefile.pre.in b/Makefile.pre.in +index f36c11d..f2b6c71 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -1217,32 +1217,6 @@ libinstall: build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c + done; \ + done + $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt +- if test -d $(DESTDIR)$(LIBDEST)/distutils/tests; then \ +- $(INSTALL_DATA) $(srcdir)/Modules/xxmodule.c \ +- $(DESTDIR)$(LIBDEST)/distutils/tests ; \ +- fi +- -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ +- $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ +- -d $(LIBDEST) -f \ +- -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \ +- $(DESTDIR)$(LIBDEST) +- -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ +- $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \ +- -d $(LIBDEST) -f \ +- -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \ +- $(DESTDIR)$(LIBDEST) +- -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ +- $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ +- -d $(LIBDEST)/site-packages -f \ +- -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages +- -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ +- $(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \ +- -d $(LIBDEST)/site-packages -f \ +- -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages +- -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ +- $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt +- -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ +- $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/PatternGrammar.txt + + # Create the PLATDIR source directory, if one wasn't distributed.. + $(srcdir)/Lib/$(PLATDIR): From f729ba98c5ba44523202e4bfe98c863830dd818c Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Sun, 19 Oct 2014 00:45:59 +0300 Subject: [PATCH 06/10] python3: export CC + CCSHARED env vars to cross-compile Python C extensions properly Signed-off-by: Alexandru Ardelean --- lang/python3/files/python3-package.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lang/python3/files/python3-package.mk b/lang/python3/files/python3-package.mk index 7808d9ecd..d5028d002 100644 --- a/lang/python3/files/python3-package.mk +++ b/lang/python3/files/python3-package.mk @@ -72,6 +72,8 @@ endef define Build/Compile/Py3Mod $(call HostPython3, \ cd $(PKG_BUILD_DIR)/$(strip $(1)); \ + CC="$(TARGET_CC)" \ + CCSHARED="$(TARGET_CC) $(FPIC)" \ CFLAGS="$(TARGET_CFLAGS)" \ CPPFLAGS="$(TARGET_CPPFLAGS)" \ LDFLAGS="$(TARGET_LDFLAGS)" \ From 3edeed3f4dc93b545f18f7b78c554d655fca51f1 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Sun, 19 Oct 2014 10:03:44 +0300 Subject: [PATCH 07/10] python3: add PYTHON3_INC_DIR var to CPPFLAGS The target's PYTHON3_INC_DIR should take precedence over the host's include dir when cross-compiling. Signed-off-by: Alexandru Ardelean --- lang/python3/files/python3-package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/python3/files/python3-package.mk b/lang/python3/files/python3-package.mk index d5028d002..ef5fbd6d7 100644 --- a/lang/python3/files/python3-package.mk +++ b/lang/python3/files/python3-package.mk @@ -75,7 +75,7 @@ define Build/Compile/Py3Mod CC="$(TARGET_CC)" \ CCSHARED="$(TARGET_CC) $(FPIC)" \ CFLAGS="$(TARGET_CFLAGS)" \ - CPPFLAGS="$(TARGET_CPPFLAGS)" \ + CPPFLAGS="$(TARGET_CPPFLAGS) -I$(PYTHON3_INC_DIR)" \ LDFLAGS="$(TARGET_LDFLAGS)" \ $(3) \ , \ From cc6423d57ab4a34dc18aa7420cc8203e65bd097d Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Sun, 19 Oct 2014 10:21:03 +0300 Subject: [PATCH 08/10] python3: add LD and LDSHARED env vars to Build/Compile/Py3Mod Tells the host Python to use the target's linker. Signed-off-by: Alexandru Ardelean --- lang/python3/files/python3-package.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lang/python3/files/python3-package.mk b/lang/python3/files/python3-package.mk index ef5fbd6d7..f29cd7f07 100644 --- a/lang/python3/files/python3-package.mk +++ b/lang/python3/files/python3-package.mk @@ -74,9 +74,11 @@ define Build/Compile/Py3Mod cd $(PKG_BUILD_DIR)/$(strip $(1)); \ CC="$(TARGET_CC)" \ CCSHARED="$(TARGET_CC) $(FPIC)" \ + LD="$(TARGET_CC)" \ + LDSHARED="$(TARGET_CC) -shared" \ CFLAGS="$(TARGET_CFLAGS)" \ CPPFLAGS="$(TARGET_CPPFLAGS) -I$(PYTHON3_INC_DIR)" \ - LDFLAGS="$(TARGET_LDFLAGS)" \ + LDFLAGS="$(TARGET_LDFLAGS) -lpython$(PYTHON3_VERSION)" \ $(3) \ , \ ./setup.py $(2) \ From 1a0e9b3786ea9721f9f76f317220b976a158f7ad Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Mon, 20 Oct 2014 18:32:41 +0300 Subject: [PATCH 09/10] python3: enable shared python3 lib; make this the norm This will reduce the bloat when users will want to compile in their Python C extensions. There will be a initial bloat (several kb) if just Python is installed, but that will be compensated when users will add more C extensions. During the build we also have to add Python's PKG_BUILD_DIR so that the shared lib is found when compiling Python's built-in C extensions. Signed-off-by: Alexandru Ardelean --- lang/python3/Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lang/python3/Makefile b/lang/python3/Makefile index d97611ea8..1e6c5c5e0 100644 --- a/lang/python3/Makefile +++ b/lang/python3/Makefile @@ -70,7 +70,7 @@ MAKE_FLAGS:=\ DESTDIR="$(PKG_INSTALL_DIR)" \ CROSS_COMPILE=yes \ CFLAGS="$(TARGET_CFLAGS) -DNDEBUG -fno-inline" \ - LDFLAGS="$(TARGET_LDFLAGS)" \ + LDFLAGS="$(TARGET_LDFLAGS) -L$(PKG_BUILD_DIR)" \ LD="$(TARGET_CC)" ENABLE_IPV6:= @@ -102,7 +102,7 @@ define Build/Configure $(CP) ./files/config.site $(PKG_BUILD_DIR) $(call Build/Configure/Default, \ --sysconfdir=/etc \ - --disable-shared \ + --enable-shared \ --without-cxx-main \ --with-threads \ --with-system-ffi="$(STAGING_DIR)/usr" \ @@ -123,7 +123,7 @@ define Build/InstallDev $(1)/usr/include/ $(CP) \ $(STAGING_DIR_HOST)/lib/python$(PYTHON_VERSION) \ - $(PKG_BUILD_DIR)/libpython$(PYTHON_VERSION).a \ + $(PKG_INSTALL_DIR)/usr/lib/libpython$(PYTHON_VERSION).so* \ $(1)/usr/lib/ $(CP) \ $(PKG_INSTALL_DIR)/usr/lib/python$(PYTHON_VERSION)/config-$(PYTHON_VERSION) \ @@ -157,6 +157,7 @@ define Py3Package/python3/install # Adding the lib-dynload folder (even just empty) suppresses 2 warnings when starting Python $(INSTALL_DIR) $(1)/usr/lib/python$(PYTHON_VERSION)/lib-dynload/ $(LN) python$(PYTHON_VERSION) $(1)/usr/bin/python3 + $(CP) $(PKG_INSTALL_DIR)/usr/lib/libpython$(PYTHON_VERSION).so* $(1)/usr/lib/ endef define Host/Configure From 056e565ab646f991ad7b3a9dd95f63ad50248f3b Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Sun, 19 Oct 2014 21:17:33 +0300 Subject: [PATCH 10/10] python3: add _PYTHON_HOST_PLATFORM env var when cross-compiling C extensions This mostly helps to avoid confusion when modules are cross-compiled. Otherwise build folders are named with the host's platform name. Signed-off-by: Alexandru Ardelean --- lang/python3/files/python3-package.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/lang/python3/files/python3-package.mk b/lang/python3/files/python3-package.mk index f29cd7f07..e2e71d3e0 100644 --- a/lang/python3/files/python3-package.mk +++ b/lang/python3/files/python3-package.mk @@ -79,6 +79,7 @@ define Build/Compile/Py3Mod CFLAGS="$(TARGET_CFLAGS)" \ CPPFLAGS="$(TARGET_CPPFLAGS) -I$(PYTHON3_INC_DIR)" \ LDFLAGS="$(TARGET_LDFLAGS) -lpython$(PYTHON3_VERSION)" \ + _PYTHON_HOST_PLATFORM="linux-$(ARCH)" \ $(3) \ , \ ./setup.py $(2) \