Merge pull request #4598 from commodo/fix-python-builds

python,python3: try to stabilize host builds

lang/python/python/Makefile

@@ -12,11 +12,10 @@ include ./files/python-version.mk
 
 PKG_NAME:=python
 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
-PKG_RELEASE:=5
+PKG_RELEASE:=6
 
 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
 PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION)
-PKG_MD5SUM:=53b43534153bb2a0363f08bae8b9d990
 PKG_HASH:=35d543986882f78261f97787fd3e06274bfa6df29fac9b4a94f73930ff98f731
 
 PKG_LICENSE:=PSF

lang/python/python/files/python-host.mk

@@ -60,6 +60,19 @@ define Build/Compile/HostPyRunHost
 	)
 endef
 
+# Note: I shamelessly copied this from Yousong's logic (from python-packages);
+HOST_PYTHON_PIP:=$(STAGING_DIR_HOSTPKG)/bin/pip$(PYTHON_VERSION)
+define host_python_pip_install
+	$(HOST_PYTHON_PIP) install \
+		--root=$(1) \
+		--prefix=$(2) \
+		--ignore-installed \
+		$(3)
+endef
+
+define host_python_pip_install_host
+$(call host_python_pip_install,$(STAGING_DIR_HOSTPKG),"",$(1))
+endef
 
 # $(1) => build subdir
 # $(2) => additional arguments to setup.py
@@ -71,11 +84,4 @@ define Build/Compile/HostPyMod
 		$(3))
 endef
 
-define HostPy/Compile/Default
-	$(call Build/Compile/HostPyMod,,\
-		install --root="$(STAGING_DIR_HOSTPKG)" --prefix="" \
-		--single-version-externally-managed \
-	)
-endef
-
 endif # __python_host_mk_inc

lang/python/python/files/python-package-install.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 process_filespec() {
 	local src_dir="$1"
@@ -50,10 +51,13 @@ find "$dst_dir" -name "*.egg-info" | xargs rm -rf
 
 if [ "$mode" == "sources" ] ; then
 	# Copy only python source files
-	find $dst_dir -not -name "*\.py" | xargs rm -f
+	find $dst_dir -type f -not -name "*\.py" | xargs rm -f
+
 	# Delete empty folders (if the case)
-	find $dst_dir/usr -type d | xargs rmdir &> /dev/null
-	rmdir $dst_dir/usr &> /dev/null
+	if [ -d "$dst_dir/usr" ] ; then
+		find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty
+		rmdir --ignore-fail-on-non-empty $dst_dir/usr
+	fi
 	exit 0
 fi
 
@@ -67,7 +71,15 @@ $python -m compileall -d '/' $dst_dir || {
 	echo "python -m compileall err-ed"
 	exit 1
 }
+
 # Delete source files and pyc [ un-optimized bytecode files ]
 # We may want to make this optimization thing configurable later, but not sure atm
-find $dst_dir -name "*\.py" | xargs rm -f
+find $dst_dir -type f -name "*\.py" | xargs rm -f
+
+# Delete empty folders (if the case)
+if [ -d "$dst_dir/usr" ] ; then
+	find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty
+	rmdir --ignore-fail-on-non-empty $dst_dir/usr
+fi
 
+exit 0

lang/python/python/files/python-package.mk

@@ -126,6 +126,9 @@ define Build/Compile/PyMod
 endef
 
 define PyBuild/Compile/Default
+	$(foreach pkg,$(HOST_PYTHON_PACKAGE_BUILD_DEPENDS),
+		$(call host_python_pip_install_host,$(pkg))
+	)
 	$(call Build/Compile/PyMod,, \
 		install --prefix="/usr" --root="$(PKG_INSTALL_DIR)" \
 		--single-version-externally-managed \

lang/python/python/patches/006-remove-debian-multiarch-support.patch

@@ -1,14 +0,0 @@
-diff --git a/setup.py b/setup.py
-index 1d1ae72..511aed5 100644
---- a/setup.py
-+++ b/setup.py
-@@ -444,7 +444,8 @@ class PyBuildExt(build_ext):
-             add_dir_to_list(self.compiler.include_dirs, '/usr/local/include')
-         if cross_compiling:
-             self.add_gcc_paths()
--        self.add_multiarch_paths()
-+        else:
-+            self.add_multiarch_paths()
- 
-         # Add paths specified in the environment variables LDFLAGS and
-         # CPPFLAGS for header and library files.

lang/python/python/patches/006-remove-multi-arch-and-local-paths.patch

@@ -0,0 +1,18 @@
+diff --git a/setup.py b/setup.py
+index 54054c2..d043761 100644
+--- a/setup.py
++++ b/setup.py
+@@ -454,13 +454,8 @@ class PyBuildExt(build_ext):
+             os.unlink(tmpfile)
+ 
+     def detect_modules(self):
+-        # Ensure that /usr/local is always used
+-        if not cross_compiling:
+-            add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib')
+-            add_dir_to_list(self.compiler.include_dirs, '/usr/local/include')
+         if cross_compiling:
+             self.add_gcc_paths()
+-        self.add_multiarch_paths()
+ 
+         # Add paths specified in the environment variables LDFLAGS and
+         # CPPFLAGS for header and library files.

lang/python/python3/Makefile

@@ -14,12 +14,11 @@ PYTHON_VERSION:=$(PYTHON3_VERSION)
 PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO)
 
 PKG_NAME:=python3
-PKG_RELEASE:=3
+PKG_RELEASE:=4
 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
 
 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
 PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION)
-PKG_MD5SUM:=692b4fc3a2ba0d54d1495d4ead5b0b5c
 PKG_HASH:=a01810ddfcec216bcdb357a84bfaafdfaa0ca42bbdaa4cb7ff74f5a9961e4041
 
 PKG_LICENSE:=PSF

lang/python/python3/files/python3-host.mk

@@ -60,6 +60,19 @@ define Build/Compile/HostPy3RunHost
 	)
 endef
 
+# Note: I shamelessly copied this from Yousong's logic (from python-packages);
+HOST_PYTHON3_PIP:=$(STAGING_DIR_HOSTPKG)/bin/pip$(PYTHON3_VERSION)
+define host_python3_pip_install
+	$(HOST_PYTHON3_PIP) install \
+		--root=$(1) \
+		--prefix=$(2) \
+		--ignore-installed \
+		$(3)
+endef
+
+define host_python3_pip_install_host
+$(call host_python3_pip_install,$(STAGING_DIR_HOSTPKG),"",$(1))
+endef
 
 # $(1) => build subdir
 # $(2) => additional arguments to setup.py
@@ -71,11 +84,4 @@ define Build/Compile/HostPy3Mod
 		$(3))
 endef
 
-define HostPy3/Compile/Default
-	$(call Build/Compile/HostPy3Mod,,\
-		install --root="$(STAGING_DIR_HOSTPKG)" --prefix="" \
-		--single-version-externally-managed \
-	)
-endef
-
 endif # __python3_host_mk_inc

lang/python/python3/files/python3-package-install.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 process_filespec() {
 	local src_dir="$1"
@@ -50,10 +51,13 @@ find "$dst_dir" -name "*.egg-info" | xargs rm -rf
 
 if [ "$mode" == "sources" ] ; then
 	# Copy only python source files
-	find $dst_dir -not -name "*\.py" | xargs rm -f
+	find $dst_dir -type f -not -name "*\.py" | xargs rm -f
+
 	# Delete empty folders (if the case)
-	find $dst_dir/usr -type d | xargs rmdir &> /dev/null
-	rmdir $dst_dir/usr &> /dev/null
+	if [ -d "$dst_dir/usr" ] ; then
+		find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty
+		rmdir --ignore-fail-on-non-empty $dst_dir/usr
+	fi
 	exit 0
 fi
 
@@ -67,6 +71,15 @@ $python -m compileall -b -d '/' $dst_dir || {
 	echo "python -m compileall err-ed"
 	exit 1
 }
+
 # Delete source files and pyc [ un-optimized bytecode files ]
 # We may want to make this optimization thing configurable later, but not sure atm
-find $dst_dir -name "*\.py" | xargs rm -f
+find $dst_dir -type f -name "*\.py" | xargs rm -f
+
+# Delete empty folders (if the case)
+if [ -d "$dst_dir/usr" ] ; then
+	find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty
+	rmdir --ignore-fail-on-non-empty $dst_dir/usr
+fi
+
+exit 0

lang/python/python3/files/python3-package.mk

@@ -126,6 +126,9 @@ define Build/Compile/Py3Mod
 endef
 
 define Py3Build/Compile/Default
+	$(foreach pkg,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS),
+		$(call host_python3_pip_install_host,$(pkg))
+	)
 	$(call Build/Compile/Py3Mod,, \
 		install --prefix="/usr" --root="$(PKG_INSTALL_DIR)" \
 		--single-version-externally-managed \

lang/python/python3/patches/006-remove-debian-multiarch-support.patch

@@ -1,12 +0,0 @@
-diff --git a/setup.py b/setup.py
-index 7868b7b..9ae0ef2 100644
---- a/setup.py
-+++ b/setup.py
-@@ -444,7 +444,6 @@ class PyBuildExt(build_ext):
-             add_dir_to_list(self.compiler.include_dirs, '/usr/local/include')
-         if cross_compiling:
-             self.add_gcc_paths()
--        self.add_multiarch_paths()
- 
-         # Add paths specified in the environment variables LDFLAGS and
-         # CPPFLAGS for header and library files.

lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch

@@ -0,0 +1,21 @@
+diff --git a/setup.py b/setup.py
+index f04bf22..01b851e 100644
+--- a/setup.py
++++ b/setup.py
+@@ -487,16 +487,9 @@ class PyBuildExt(build_ext):
+             return ['m']
+ 
+     def detect_modules(self):
+-        # Ensure that /usr/local is always used, but the local build
+-        # directories (i.e. '.' and 'Include') must be first.  See issue
+-        # 10520.
+-        if not cross_compiling:
+-            add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib')
+-            add_dir_to_list(self.compiler.include_dirs, '/usr/local/include')
+         # only change this for cross builds for 3.3, issues on Mageia
+         if cross_compiling:
+             self.add_gcc_paths()
+-        self.add_multiarch_paths()
+ 
+         # Add paths specified in the environment variables LDFLAGS and
+         # CPPFLAGS for header and library files.