python3-packages: port from python-packages

This is largely done by suffixing "python" or "py" with "3".  The
README.md file is also copied here and we intend to maintain it
independently from its python2 counterpart.

Signed-off-by: Yousong Zhou <yszhou4tech@gmail.com>

lang/python/python3-packages/Makefile

@@ -0,0 +1,134 @@
+#
+# Copyright (C) 2019 Yousong Zhou <yszhou4tech@gmail.com>
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+
+include $(TOPDIR)/rules.mk
+
+PKG_NAME:=python3-packages
+PKG_VERSION:=1.0
+PKG_RELEASE:=1
+
+PKG_MAINTAINER:=Yousong Zhou <yszhou4tech@gmail.com>
+
+#
+# NOTE: move the host module installation to Host/Compile when
+# HOST_CONFIG_DEPENDS is supported
+#
+# NOTE: PKG_CONFIG_DEPENDS cannot correctly track changes of string type config
+# options, so you may want to do manual cleanup on config change.
+#
+PKG_CONFIG_DEPENDS:= \
+	CONFIG_PACKAGE_python3-packages-list-host \
+	CONFIG_PACKAGE_python3-packages-list \
+	CONFIG_PACKAGE_python3-packages-list-cleanup \
+	CONFIG_PACKAGE_python3-packages-envs \
+	CONFIG_PACKAGE_python3-packages-extra-deps \
+	CONFIG_PACKAGE_python3-packages-index-url \
+	CONFIG_PACKAGE_python3-packages-pip-opts \
+
+PKG_BUILD_DEPENDS:=python3 python3/host
+
+include $(INCLUDE_DIR)/package.mk
+include ../python3-host.mk
+include ../python3-package.mk
+
+define Package/python3-packages
+  SUBMENU:=Python
+  SECTION:=lang
+  CATEGORY:=Languages
+  TITLE:=A dummy package for packaging python3 modules with pip
+  DEPENDS:=@DEVEL +python3
+endef
+
+define Package/python3-packages/config
+if PACKAGE_python3-packages
+config PACKAGE_python3-packages-list-host
+	string "List of python3 packages to install on host"
+config PACKAGE_python3-packages-list
+	string "List of python3 packages to install on target"
+config PACKAGE_python3-packages-list-cleanup
+	string "List of python3 packages to cleanup to avoid clash with existing packages"
+config PACKAGE_python3-packages-envs
+	string "Extra environment variables to pass on to pip and its children on target build"
+config PACKAGE_python3-packages-extra-deps
+	string "List of deps fulfilled but not tracked by the build system"
+config PACKAGE_python3-packages-index-url
+	string "Index URL passed to pip with --index-url"
+config PACKAGE_python3-packages-pip-opts
+	string "Additional arguments to pip command line"
+endif
+endef
+
+CONFIG_PACKAGE_python3-packages-list-host:=$(call qstrip,$(CONFIG_PACKAGE_python3-packages-list-host))
+CONFIG_PACKAGE_python3-packages-list:=$(call qstrip,$(CONFIG_PACKAGE_python3-packages-list))
+CONFIG_PACKAGE_python3-packages-list-cleanup:=$(call qstrip,$(CONFIG_PACKAGE_python3-packages-list-cleanup))
+CONFIG_PACKAGE_python3-packages-envs:=$(call qstrip,$(CONFIG_PACKAGE_python3-packages-envs))
+CONFIG_PACKAGE_python3-packages-extra-deps:=$(call qstrip,$(CONFIG_PACKAGE_python3-packages-extra-deps))
+CONFIG_PACKAGE_python3-packages-index-url:=$(call qstrip,$(CONFIG_PACKAGE_python3-packages-index-url))
+CONFIG_PACKAGE_python3-packages-pip-opts:=$(call qstrip,$(CONFIG_PACKAGE_python3-packages-pip-opts))
+
+decr=$(word $(1),0 1 2 3 4 5 6 7 8 9 10)
+recur=$(if $(subst 0,,$(2)),$(call recur,$(1),$(call decr,$(2)),$(call $(1)$(2),$(3))),$(3))
+_req2dir1=$(subst >,gt,$(1))
+_req2dir2=$(subst <,lt,$(1))
+_req2dir3=$(subst >=,geq,$(1))
+_req2dir4=$(subst <=,leq,$(1))
+_req2dir5=$(subst ://,:::,$(1))
+_req2dir6=$(subst *,_,$(1))
+_req2dir7=$(subst ?,_,$(1))
+req2dir=$(call recur,_req2dir,7,$(1))
+
+# --ignore-installed, it may happen that host pip will ignore target install if
+#  it was already installed as host module, e.g. cffi deps of cryptograph
+#
+# --no-build-isolation, this is a new addition in pip3 and is needed when build
+#  depends on host modules
+HOST_PYTHON3_PIP_INSTALL=$(HOST_PYTHON3_PIP) install \
+	--root=$(1) \
+	--prefix=$(2) \
+	--ignore-installed \
+	--no-build-isolation \
+	--no-compile \
+	$(if $(CONFIG_PACKAGE_python3-packages-index-url), --index-url "$(CONFIG_PACKAGE_python3-packages-index-url)") \
+	$(if $(CONFIG_PACKAGE_python3-packages-pip-opts), $(CONFIG_PACKAGE_python3-packages-pip-opts)) \
+
+HOST_PYTHON3_PIP_INSTALL_HOST:=$(call HOST_PYTHON3_PIP_INSTALL,$(STAGING_DIR_HOSTPKG),"")
+HOST_PYTHON3_PIP_INSTALL_TARGET=$(call HOST_PYTHON3_PIP_INSTALL,$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg)),/usr)
+HOST_PYTHON3_PIP_INSTALL_CLEANUP:=$(call HOST_PYTHON3_PIP_INSTALL,$(PKG_INSTALL_DIR)/_cleanup,/usr)
+
+define Build/Compile
+	$(foreach pkg,$(CONFIG_PACKAGE_python3-packages-list-host),
+		$(call Build/Compile/HostPy3RunHost,,$(HOST_PYTHON3_PIP_INSTALL_HOST) $(pkg))
+	)
+	$(foreach pkg,$(CONFIG_PACKAGE_python3-packages-list),
+		$(call Build/Compile/HostPy3RunTarget,,$(call HOST_PYTHON3_PIP_INSTALL_TARGET,$(pkg)) $(pkg),$(CONFIG_PACKAGE_python3-packages-envs))
+	)
+	$(foreach pkg,$(CONFIG_PACKAGE_python3-packages-list-cleanup),
+		$(call Build/Compile/HostPy3RunTarget,,$(HOST_PYTHON3_PIP_INSTALL_CLEANUP) $(pkg),$(CONFIG_PACKAGE_python3-packages-envs))
+	)
+endef
+
+define Package/python3-packages/install
+	$(foreach pkg,$(CONFIG_PACKAGE_python3-packages-list),
+		$(CP) "$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg))"/* $(1)
+	)
+
+	find "$(PKG_INSTALL_DIR)/_cleanup" -mindepth 1 -depth | while read sf; do \
+		tf="$$$${sf#$(PKG_INSTALL_DIR)/_cleanup/}"; \
+		tf="$(1)/$$$$tf"; \
+		if [ -f "$$$$tf" -o -L "$$$$tf" ]; then \
+			rm -vf "$$$$tf"; \
+		elif [ -d "$$$$tf" ]; then \
+			rmdir -v -p "$$$$tf" || true; \
+		fi \
+	done
+endef
+
+define Package/python3-packages/extra_provides
+	echo $(CONFIG_PACKAGE_python3-packages-extra-deps) | tr ' ' '\n'
+endef
+
+$(eval $(call BuildPackage,python3-packages))

lang/python/python3-packages/README.md

@@ -0,0 +1,72 @@
+This package allows users to package python3 modules without creating package
+Makefiles for each individual module and their dependencies.  It provides a
+way making packaging python3 packages faster and may also facilitate the process
+of developing Makefiles for new python3 packages
+
+This is a raw DEVEL only package.  Using it may entails a lot of implementation
+details and you may need to resolve target dependencies and package details on
+your own
+
+- Third party python3 packages may depend on features not included in e.g.
+  python3-light
+- Some python3 modules may require host install of another module to progress,
+  e.g. target cryptography requires host cffi
+- Some python3 modules have external C library dependencies, e.g. pyOpenSSL
+  requires openssl libs
+- Some packages may have an autoconf configure script whose arguments we
+  cannot control with pip and has to be passed on (hacked) by overriding some
+  environment variables
+
+## How it works
+
+1. Install host modules required for building target modules
+2. Install each target module to separate directories
+3. Install another copy of modules for cleanup purposes to make list of
+   installed files to be removed from target modules installed in step 2
+
+Why should it be so
+
+1. Installing target cryptography requires host installation of cffi module
+2. cryptography requires setuptools and pip will install its own copy with
+   --ignore-installed.  When PACKAGE_python3-setuptools is also selected, opkg
+   will complain of data file clashes if it was not removed here.
+
+Pip will handle dependency requirements of python3 modules, but external
+dependencies like c libraries has to be prepared by the build system.  The
+issue is that there is currently no way to express such dependencies, thus may
+cause build failure, e.g. pycrypto requires the presence of libgmp to build
+successfully.
+
+## Tips
+
+If something goes wrong, we can add additional arguments to pip command
+line to check the detailed build process.  Some useful arguments may be
+
+- -v, for verbose output.  Repeat this option if the current level of
+  verbosity is not enough
+- --no-clean, for preserving pip build dir on build failure
+
+## Examples
+
+tornado (python-only module)
+
+	CONFIG_PACKAGE_python3-packages=y
+	CONFIG_PACKAGE_python3-packages-list="tornado==6.0.2"
+
+cryptography (requires installation of host modules and cleanup on target modules)
+
+	CONFIG_PACKAGE_python3-packages=y
+	CONFIG_PACKAGE_python3-packages-list-host="cffi"
+	CONFIG_PACKAGE_python3-packages-list="cryptography"
+	CONFIG_PACKAGE_python3-packages-list-cleanup="setuptools"
+
+pycrypto 2.7a1 (python module with autoconf configure script; depends on
+libgmp; broken wmmintrin.h).  2.6.1 does not work because of a flaw in
+the setup.py hardcoding host include directory
+
+	CONFIG_PACKAGE_libgmp=y
+	CONFIG_PACKAGE_python3-packages=y
+	CONFIG_PACKAGE_python3-packages-list="https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz"
+	CONFIG_PACKAGE_python3-packages-envs="ac_cv_header_wmmintrin_h=no build_alias=$(GNU_HOST_NAME) host_alias=$(GNU_TARGET_NAME) target_alias=$(GNU_TARGET_NAME)"
+	CONFIG_PACKAGE_python3-packages-extra-deps="libgmp.so.10"
+