python-setuptools: Add reproducibility patches from Debian

This adds the ability to patch setuptools (and pip), and adds 3
reproducibility patches from Debian[1].
(003-PKG-INFO-output-reproducible.patch addresses the issue identified
in #9039.)

The patching is not perfect, in that the patches are applied to
setuptools and pip after they have been installed, since they are
installed from wheels which are already "precompiled".

Also, patching for the host install cannot be updated in place, for
example if a patch is added or removed.

[1]: https://sources.debian.org/patches/python-setuptools/40.8.0-1/

Signed-off-by: Jeffery To <jeffery.to@gmail.com>

lang/python/python-version.mk

@@ -8,7 +8,7 @@
 PYTHON_VERSION:=2.7
 PYTHON_VERSION_MICRO:=16
 
-PYTHON_SETUPTOOLS_PKG_RELEASE:=2
+PYTHON_SETUPTOOLS_PKG_RELEASE:=3
 PYTHON_PIP_PKG_RELEASE:=2
 
 PYTHON_SETUPTOOLS_VERSION:=40.6.2

lang/python/python/Makefile

@@ -174,6 +174,7 @@ define Build/Compile/python-setuptools
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON_SETUPTOOLS_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON_VERSION)/site-packages,./patches-setuptools,)
 endef
 endif # CONFIG_PACKAGE_python-setuptools
 
@@ -183,6 +184,7 @@ define Build/Compile/python-pip
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-pip --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON_PIP_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON_VERSION)/site-packages,./patches-pip,)
 endef
 endif # CONFIG_PACKAGE_python-pip
 
@@ -304,6 +306,14 @@ define Host/Install
 	$(MAKE) -C $(HOST_BUILD_DIR) install
 	$(INSTALL_DIR) $(HOST_PYTHON_DIR)/bin/
 	$(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON_DIR)/bin/pgen2
+  ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.setuptools-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-setuptools,)
+	touch $(HOST_PYTHON_PKG_DIR)/.setuptools-patched
+  endif
+  ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.pip-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-pip,)
+	touch $(HOST_PYTHON_PKG_DIR)/.pip-patched
+  endif
 endef
 
 $(eval $(call HostBuild))

lang/python/python/patches-setuptools/001-reproducible.patch

@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/
+
+Index: b/setuptools/command/easy_install.py
+===================================================================
+--- a/setuptools/command/easy_install.py
++++ b/setuptools/command/easy_install.py
+@@ -436,7 +436,7 @@ consider to install to another location,
+             for spec in self.args:
+                 self.easy_install(spec, not self.no_deps)
+             if self.record:
+-                outputs = self.outputs
++                outputs = list(sorted(self.outputs))
+                 if self.root:  # strip any package prefix
+                     root_len = len(self.root)
+                     for counter in range(len(outputs)):

lang/python/python/patches-setuptools/002-sorted-requires.patch

@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/
+
+Index: b/setuptools/command/egg_info.py
+===================================================================
+--- a/setuptools/command/egg_info.py
++++ b/setuptools/command/egg_info.py
+@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename,
+ def _write_requirements(stream, reqs):
+     lines = yield_lines(reqs or ())
+     append_cr = lambda line: line + '\n'
+-    lines = map(append_cr, lines)
++    lines = map(append_cr, sorted(lines))
+     stream.writelines(lines)
+ 
+ 

lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch

@@ -0,0 +1,14 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/
+
+--- a/setuptools/dist.py
++++ b/setuptools/dist.py
+@@ -191,7 +191,7 @@ def write_pkg_file(self, file):
+             self.long_description_content_type
+         )
+     if self.provides_extras:
+-        for extra in self.provides_extras:
++        for extra in sorted(self.provides_extras):
+             write_field('Provides-Extra', extra)
+ 
+ 

lang/python/python3-version.mk

@@ -12,7 +12,7 @@ PYTHON3_VERSION_MICRO:=2
 
 PYTHON3_VERSION:=$(PYTHON3_VERSION_MAJOR).$(PYTHON3_VERSION_MINOR)
 
-PYTHON3_SETUPTOOLS_PKG_RELEASE:=2
+PYTHON3_SETUPTOOLS_PKG_RELEASE:=3
 PYTHON3_PIP_PKG_RELEASE:=2
 
 PYTHON3_SETUPTOOLS_VERSION:=40.6.2

lang/python/python3/Makefile

@@ -178,6 +178,7 @@ define Build/Compile/python3-setuptools
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON3_SETUPTOOLS_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON3_VERSION)/site-packages,./patches-setuptools,)
 endef
 endif # CONFIG_PACKAGE_python3-setuptools
 
@@ -187,6 +188,7 @@ define Build/Compile/python3-pip
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-pip --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON3_PIP_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON3_VERSION)/site-packages,./patches-pip,)
 endef
 endif # CONFIG_PACKAGE_python3-pip
 
@@ -299,6 +301,14 @@ define Host/Install
 	$(MAKE) -C $(HOST_BUILD_DIR) install
 	$(INSTALL_DIR) $(HOST_PYTHON3_DIR)/bin/
 	$(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON3_DIR)/bin/pgen3
+  ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-setuptools,)
+	touch $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched
+  endif
+  ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.pip-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-pip,)
+	touch $(HOST_PYTHON3_PKG_DIR)/.pip-patched
+  endif
 endef
 
 $(eval $(call HostBuild))

lang/python/python3/patches-setuptools/001-reproducible.patch

@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/
+
+Index: b/setuptools/command/easy_install.py
+===================================================================
+--- a/setuptools/command/easy_install.py
++++ b/setuptools/command/easy_install.py
+@@ -436,7 +436,7 @@ consider to install to another location,
+             for spec in self.args:
+                 self.easy_install(spec, not self.no_deps)
+             if self.record:
+-                outputs = self.outputs
++                outputs = list(sorted(self.outputs))
+                 if self.root:  # strip any package prefix
+                     root_len = len(self.root)
+                     for counter in range(len(outputs)):

lang/python/python3/patches-setuptools/002-sorted-requires.patch

@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/
+
+Index: b/setuptools/command/egg_info.py
+===================================================================
+--- a/setuptools/command/egg_info.py
++++ b/setuptools/command/egg_info.py
+@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename,
+ def _write_requirements(stream, reqs):
+     lines = yield_lines(reqs or ())
+     append_cr = lambda line: line + '\n'
+-    lines = map(append_cr, lines)
++    lines = map(append_cr, sorted(lines))
+     stream.writelines(lines)
+ 
+ 

lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch

@@ -0,0 +1,14 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/
+
+--- a/setuptools/dist.py
++++ b/setuptools/dist.py
+@@ -191,7 +191,7 @@ def write_pkg_file(self, file):
+             self.long_description_content_type
+         )
+     if self.provides_extras:
+-        for extra in self.provides_extras:
++        for extra in sorted(self.provides_extras):
+             write_field('Provides-Extra', extra)
+ 
+