From e066fdaf1147c0ce361ee1f08a4e56170be26e1b Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Sun, 16 Jul 2017 18:46:36 +0300 Subject: [PATCH 1/5] python,python3: handle install script errors better Depending on execution order the `python-package-install.sh` script would return a non-zero err code. So, this enforces that all commands in the script don't fail (via the `set -e` directive). Signed-off-by: Alexandru Ardelean --- .../python/files/python-package-install.sh | 20 ++++++++++++++---- .../python3/files/python3-package-install.sh | 21 +++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/lang/python/python/files/python-package-install.sh b/lang/python/python/files/python-package-install.sh index 022cf8a35..e8085e5e3 100644 --- a/lang/python/python/files/python-package-install.sh +++ b/lang/python/python/files/python-package-install.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e process_filespec() { local src_dir="$1" @@ -50,10 +51,13 @@ find "$dst_dir" -name "*.egg-info" | xargs rm -rf if [ "$mode" == "sources" ] ; then # Copy only python source files - find $dst_dir -not -name "*\.py" | xargs rm -f + find $dst_dir -type f -not -name "*\.py" | xargs rm -f + # Delete empty folders (if the case) - find $dst_dir/usr -type d | xargs rmdir &> /dev/null - rmdir $dst_dir/usr &> /dev/null + if [ -d "$dst_dir/usr" ] ; then + find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty + rmdir --ignore-fail-on-non-empty $dst_dir/usr + fi exit 0 fi @@ -67,7 +71,15 @@ $python -m compileall -d '/' $dst_dir || { echo "python -m compileall err-ed" exit 1 } + # Delete source files and pyc [ un-optimized bytecode files ] # We may want to make this optimization thing configurable later, but not sure atm -find $dst_dir -name "*\.py" | xargs rm -f +find $dst_dir -type f -name "*\.py" | xargs rm -f + +# Delete empty folders (if the case) +if [ -d "$dst_dir/usr" ] ; then + find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty + rmdir --ignore-fail-on-non-empty $dst_dir/usr +fi +exit 0 diff --git a/lang/python/python3/files/python3-package-install.sh b/lang/python/python3/files/python3-package-install.sh index ae5e17280..cbb84ab1c 100644 --- a/lang/python/python3/files/python3-package-install.sh +++ b/lang/python/python3/files/python3-package-install.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e process_filespec() { local src_dir="$1" @@ -50,10 +51,13 @@ find "$dst_dir" -name "*.egg-info" | xargs rm -rf if [ "$mode" == "sources" ] ; then # Copy only python source files - find $dst_dir -not -name "*\.py" | xargs rm -f + find $dst_dir -type f -not -name "*\.py" | xargs rm -f + # Delete empty folders (if the case) - find $dst_dir/usr -type d | xargs rmdir &> /dev/null - rmdir $dst_dir/usr &> /dev/null + if [ -d "$dst_dir/usr" ] ; then + find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty + rmdir --ignore-fail-on-non-empty $dst_dir/usr + fi exit 0 fi @@ -67,6 +71,15 @@ $python -m compileall -b -d '/' $dst_dir || { echo "python -m compileall err-ed" exit 1 } + # Delete source files and pyc [ un-optimized bytecode files ] # We may want to make this optimization thing configurable later, but not sure atm -find $dst_dir -name "*\.py" | xargs rm -f +find $dst_dir -type f -name "*\.py" | xargs rm -f + +# Delete empty folders (if the case) +if [ -d "$dst_dir/usr" ] ; then + find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty + rmdir --ignore-fail-on-non-empty $dst_dir/usr +fi + +exit 0 From 84202f17e1aac6faf66b8d186f7c5c62b6f72ffb Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Sun, 16 Jul 2017 23:31:37 +0300 Subject: [PATCH 2/5] python,python3: remove multi-arch support, and /usr/local paths This should hopefully reduce weird behaviour caused by the host system. Signed-off-by: Alexandru Ardelean --- lang/python/python/Makefile | 2 +- .../006-remove-debian-multiarch-support.patch | 14 ------------- ...06-remove-multi-arch-and-local-paths.patch | 18 ++++++++++++++++ lang/python/python3/Makefile | 2 +- .../006-remove-debian-multiarch-support.patch | 12 ----------- ...06-remove-multi-arch-and-local-paths.patch | 21 +++++++++++++++++++ 6 files changed, 41 insertions(+), 28 deletions(-) delete mode 100644 lang/python/python/patches/006-remove-debian-multiarch-support.patch create mode 100644 lang/python/python/patches/006-remove-multi-arch-and-local-paths.patch delete mode 100644 lang/python/python3/patches/006-remove-debian-multiarch-support.patch create mode 100644 lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch diff --git a/lang/python/python/Makefile b/lang/python/python/Makefile index 09269caeb..60e7fe19c 100644 --- a/lang/python/python/Makefile +++ b/lang/python/python/Makefile @@ -12,7 +12,7 @@ include ./files/python-version.mk PKG_NAME:=python PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO) -PKG_RELEASE:=5 +PKG_RELEASE:=6 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION) diff --git a/lang/python/python/patches/006-remove-debian-multiarch-support.patch b/lang/python/python/patches/006-remove-debian-multiarch-support.patch deleted file mode 100644 index 01aa924ab..000000000 --- a/lang/python/python/patches/006-remove-debian-multiarch-support.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/setup.py b/setup.py -index 1d1ae72..511aed5 100644 ---- a/setup.py -+++ b/setup.py -@@ -444,7 +444,8 @@ class PyBuildExt(build_ext): - add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') - if cross_compiling: - self.add_gcc_paths() -- self.add_multiarch_paths() -+ else: -+ self.add_multiarch_paths() - - # Add paths specified in the environment variables LDFLAGS and - # CPPFLAGS for header and library files. diff --git a/lang/python/python/patches/006-remove-multi-arch-and-local-paths.patch b/lang/python/python/patches/006-remove-multi-arch-and-local-paths.patch new file mode 100644 index 000000000..1ecc153fa --- /dev/null +++ b/lang/python/python/patches/006-remove-multi-arch-and-local-paths.patch @@ -0,0 +1,18 @@ +diff --git a/setup.py b/setup.py +index 54054c2..d043761 100644 +--- a/setup.py ++++ b/setup.py +@@ -454,13 +454,8 @@ class PyBuildExt(build_ext): + os.unlink(tmpfile) + + def detect_modules(self): +- # Ensure that /usr/local is always used +- if not cross_compiling: +- add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') +- add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + if cross_compiling: + self.add_gcc_paths() +- self.add_multiarch_paths() + + # Add paths specified in the environment variables LDFLAGS and + # CPPFLAGS for header and library files. diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile index 02c103193..98e32b16e 100644 --- a/lang/python/python3/Makefile +++ b/lang/python/python3/Makefile @@ -14,7 +14,7 @@ PYTHON_VERSION:=$(PYTHON3_VERSION) PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO) PKG_NAME:=python3 -PKG_RELEASE:=3 +PKG_RELEASE:=4 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO) PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz diff --git a/lang/python/python3/patches/006-remove-debian-multiarch-support.patch b/lang/python/python3/patches/006-remove-debian-multiarch-support.patch deleted file mode 100644 index 52d52b94e..000000000 --- a/lang/python/python3/patches/006-remove-debian-multiarch-support.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/setup.py b/setup.py -index 7868b7b..9ae0ef2 100644 ---- a/setup.py -+++ b/setup.py -@@ -444,7 +444,6 @@ class PyBuildExt(build_ext): - add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') - if cross_compiling: - self.add_gcc_paths() -- self.add_multiarch_paths() - - # Add paths specified in the environment variables LDFLAGS and - # CPPFLAGS for header and library files. diff --git a/lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch b/lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch new file mode 100644 index 000000000..ed2fdb56a --- /dev/null +++ b/lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch @@ -0,0 +1,21 @@ +diff --git a/setup.py b/setup.py +index f04bf22..01b851e 100644 +--- a/setup.py ++++ b/setup.py +@@ -487,16 +487,9 @@ class PyBuildExt(build_ext): + return ['m'] + + def detect_modules(self): +- # Ensure that /usr/local is always used, but the local build +- # directories (i.e. '.' and 'Include') must be first. See issue +- # 10520. +- if not cross_compiling: +- add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') +- add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + # only change this for cross builds for 3.3, issues on Mageia + if cross_compiling: + self.add_gcc_paths() +- self.add_multiarch_paths() + + # Add paths specified in the environment variables LDFLAGS and + # CPPFLAGS for header and library files. From 0df253cc04358b7c061bd5c1564db0dcd8179fd6 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Wed, 19 Jul 2017 09:40:34 +0300 Subject: [PATCH 3/5] python,python3: drop PKG_MD5SUM fields ; will use only PKG_HASH Signed-off-by: Alexandru Ardelean --- lang/python/python/Makefile | 1 - lang/python/python3/Makefile | 1 - 2 files changed, 2 deletions(-) diff --git a/lang/python/python/Makefile b/lang/python/python/Makefile index 60e7fe19c..a516b03bd 100644 --- a/lang/python/python/Makefile +++ b/lang/python/python/Makefile @@ -16,7 +16,6 @@ PKG_RELEASE:=6 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION) -PKG_MD5SUM:=53b43534153bb2a0363f08bae8b9d990 PKG_HASH:=35d543986882f78261f97787fd3e06274bfa6df29fac9b4a94f73930ff98f731 PKG_LICENSE:=PSF diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile index 98e32b16e..053c003e8 100644 --- a/lang/python/python3/Makefile +++ b/lang/python/python3/Makefile @@ -19,7 +19,6 @@ PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO) PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION) -PKG_MD5SUM:=692b4fc3a2ba0d54d1495d4ead5b0b5c PKG_HASH:=a01810ddfcec216bcdb357a84bfaafdfaa0ca42bbdaa4cb7ff74f5a9961e4041 PKG_LICENSE:=PSF From 93a6545591cb5208f297c0b95d56470ae26f45b3 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Wed, 19 Jul 2017 10:07:35 +0300 Subject: [PATCH 4/5] python,python3: drop HostPy/Compile/Default rules Not used, and will not be used. Signed-off-by: Alexandru Ardelean --- lang/python/python/files/python-host.mk | 7 ------- lang/python/python3/files/python3-host.mk | 7 ------- 2 files changed, 14 deletions(-) diff --git a/lang/python/python/files/python-host.mk b/lang/python/python/files/python-host.mk index 6953a09d9..17e0758dd 100644 --- a/lang/python/python/files/python-host.mk +++ b/lang/python/python/files/python-host.mk @@ -71,11 +71,4 @@ define Build/Compile/HostPyMod $(3)) endef -define HostPy/Compile/Default - $(call Build/Compile/HostPyMod,,\ - install --root="$(STAGING_DIR_HOSTPKG)" --prefix="" \ - --single-version-externally-managed \ - ) -endef - endif # __python_host_mk_inc diff --git a/lang/python/python3/files/python3-host.mk b/lang/python/python3/files/python3-host.mk index 3abf6aa5f..487aada30 100644 --- a/lang/python/python3/files/python3-host.mk +++ b/lang/python/python3/files/python3-host.mk @@ -71,11 +71,4 @@ define Build/Compile/HostPy3Mod $(3)) endef -define HostPy3/Compile/Default - $(call Build/Compile/HostPy3Mod,,\ - install --root="$(STAGING_DIR_HOSTPKG)" --prefix="" \ - --single-version-externally-managed \ - ) -endef - endif # __python3_host_mk_inc From 612c53fc6c3d9ba2a57f7329baf055f1d59a9246 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Wed, 19 Jul 2017 10:19:10 +0300 Subject: [PATCH 5/5] python,python3: add host python pip install support To install Python packages host side, that may be needed for a build. The intent, is to try to reduce host-side Python packages being installed via LEDE/OpenWrt build system. Because those seem like a pain to maintain. The idea is adapted from Yousong's `python-packages` package. Signed-off-by: Alexandru Ardelean --- lang/python/python/files/python-host.mk | 13 +++++++++++++ lang/python/python/files/python-package.mk | 3 +++ lang/python/python3/files/python3-host.mk | 13 +++++++++++++ lang/python/python3/files/python3-package.mk | 3 +++ 4 files changed, 32 insertions(+) diff --git a/lang/python/python/files/python-host.mk b/lang/python/python/files/python-host.mk index 17e0758dd..f20dbdeab 100644 --- a/lang/python/python/files/python-host.mk +++ b/lang/python/python/files/python-host.mk @@ -60,6 +60,19 @@ define Build/Compile/HostPyRunHost ) endef +# Note: I shamelessly copied this from Yousong's logic (from python-packages); +HOST_PYTHON_PIP:=$(STAGING_DIR_HOSTPKG)/bin/pip$(PYTHON_VERSION) +define host_python_pip_install + $(HOST_PYTHON_PIP) install \ + --root=$(1) \ + --prefix=$(2) \ + --ignore-installed \ + $(3) +endef + +define host_python_pip_install_host +$(call host_python_pip_install,$(STAGING_DIR_HOSTPKG),"",$(1)) +endef # $(1) => build subdir # $(2) => additional arguments to setup.py diff --git a/lang/python/python/files/python-package.mk b/lang/python/python/files/python-package.mk index 1b82352c1..9c231f6ec 100644 --- a/lang/python/python/files/python-package.mk +++ b/lang/python/python/files/python-package.mk @@ -126,6 +126,9 @@ define Build/Compile/PyMod endef define PyBuild/Compile/Default + $(foreach pkg,$(HOST_PYTHON_PACKAGE_BUILD_DEPENDS), + $(call host_python_pip_install_host,$(pkg)) + ) $(call Build/Compile/PyMod,, \ install --prefix="/usr" --root="$(PKG_INSTALL_DIR)" \ --single-version-externally-managed \ diff --git a/lang/python/python3/files/python3-host.mk b/lang/python/python3/files/python3-host.mk index 487aada30..96bbc19cc 100644 --- a/lang/python/python3/files/python3-host.mk +++ b/lang/python/python3/files/python3-host.mk @@ -60,6 +60,19 @@ define Build/Compile/HostPy3RunHost ) endef +# Note: I shamelessly copied this from Yousong's logic (from python-packages); +HOST_PYTHON3_PIP:=$(STAGING_DIR_HOSTPKG)/bin/pip$(PYTHON3_VERSION) +define host_python3_pip_install + $(HOST_PYTHON3_PIP) install \ + --root=$(1) \ + --prefix=$(2) \ + --ignore-installed \ + $(3) +endef + +define host_python3_pip_install_host +$(call host_python3_pip_install,$(STAGING_DIR_HOSTPKG),"",$(1)) +endef # $(1) => build subdir # $(2) => additional arguments to setup.py diff --git a/lang/python/python3/files/python3-package.mk b/lang/python/python3/files/python3-package.mk index 22ee527a1..93b14fac2 100644 --- a/lang/python/python3/files/python3-package.mk +++ b/lang/python/python3/files/python3-package.mk @@ -126,6 +126,9 @@ define Build/Compile/Py3Mod endef define Py3Build/Compile/Default + $(foreach pkg,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS), + $(call host_python3_pip_install_host,$(pkg)) + ) $(call Build/Compile/Py3Mod,, \ install --prefix="/usr" --root="$(PKG_INSTALL_DIR)" \ --single-version-externally-managed \