diff --git a/lang/python-packages/Makefile b/lang/python-packages/Makefile new file mode 100644 index 000000000..f12b78e4d --- /dev/null +++ b/lang/python-packages/Makefile @@ -0,0 +1,131 @@ +# +# Copyright (C) 2016 Yousong Zhou +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=python-packages +PKG_VERSION:=1.0 +PKG_RELEASE:=1 + +PKG_MAINTAINER:=Yousong Zhou + +# +# NOTE: move the host module installation to Host/Compile when +# HOST_CONFIG_DEPENDS is supported +# +# NOTE: PKG_CONFIG_DEPENDS cannot correctly track changes of string type config +# options, so you may want to do manual cleanup on config change. +# +PKG_CONFIG_DEPENDS:= \ + CONFIG_PACKAGE_python-packages-list-host \ + CONFIG_PACKAGE_python-packages-list \ + CONFIG_PACKAGE_python-packages-list-cleanup \ + CONFIG_PACKAGE_python-packages-envs \ + CONFIG_PACKAGE_python-packages-extra-deps \ + CONFIG_PACKAGE_python-packages-index-url \ + CONFIG_PACKAGE_python-packages-pip-opts \ + +PKG_BUILD_DEPENDS:=python python-pip/host + +include $(INCLUDE_DIR)/package.mk +$(call include_mk, python-host.mk) +$(call include_mk, python-package.mk) + +define Package/python-packages + SUBMENU:=Python + SECTION:=lang + CATEGORY:=Languages + TITLE:=A dummy package for packaging python modules with pip + DEPENDS:=@DEVEL +python +endef + +define Package/python-packages/config +if PACKAGE_python-packages +config PACKAGE_python-packages-list-host + string "List of python packages to install on host" +config PACKAGE_python-packages-list + string "List of python packages to install on target" +config PACKAGE_python-packages-list-cleanup + string "List of python packages to cleanup to avoid clash with existing packages" +config PACKAGE_python-packages-envs + string "Extra environment variables to pass on to pip and its children on target build" +config PACKAGE_python-packages-extra-deps + string "List of deps fulfilled but not tracked by the build system" +config PACKAGE_python-packages-index-url + string "Index URL passed to pip with --index-url" +config PACKAGE_python-packages-pip-opts + string "Additional arguments to pip command line" +endif +endef + +CONFIG_PACKAGE_python-packages-list-host:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-host)) +CONFIG_PACKAGE_python-packages-list:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list)) +CONFIG_PACKAGE_python-packages-list-cleanup:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-cleanup)) +CONFIG_PACKAGE_python-packages-envs:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-envs)) +CONFIG_PACKAGE_python-packages-extra-deps:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-extra-deps)) +CONFIG_PACKAGE_python-packages-pip-opts:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-pip-opts)) + +HOST_PYTHON_PIP:=$(STAGING_DIR)/host/bin/pip$(PYTHON_VERSION) + +decr=$(word $(1),0 1 2 3 4 5 6 7 8 9 10) +recur=$(if $(subst 0,,$(2)),$(call recur,$(1),$(call decr,$(2)),$(call $(1)$(2),$(3))),$(3)) +_req2dir1=$(subst >,gt,$(1)) +_req2dir2=$(subst <,lt,$(1)) +_req2dir3=$(subst >=,geq,$(1)) +_req2dir4=$(subst <=,leq,$(1)) +_req2dir5=$(subst ://,:::,$(1)) +_req2dir6=$(subst *,_,$(1)) +_req2dir7=$(subst ?,_,$(1)) +req2dir=$(call recur,_req2dir,7,$(1)) + +# --ignore-installed, it may happen that host pip will ignore target install if +# it was already installed as host module, e.g. cffi deps of cryptograph +HOST_PYTHON_PIP_INSTALL=$(HOST_PYTHON_PIP) install \ + --root=$(1) \ + --prefix=$(2) \ + --ignore-installed \ + --no-compile \ + $(if $(CONFIG_PACKAGE_python-packages-index-url), --index-url $(CONFIG_PACKAGE_python-packages-index-url)) \ + $(if $(CONFIG_PACKAGE_python-packages-pip-opts), $(CONFIG_PACKAGE_python-packages-pip-opts)) \ + +HOST_PYTHON_PIP_INSTALL_HOST:=$(call HOST_PYTHON_PIP_INSTALL,$(STAGING_DIR)/host,"") +HOST_PYTHON_PIP_INSTALL_TARGET=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg)),/usr) +HOST_PYTHON_PIP_INSTALL_CLEANUP:=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/_cleanup,/usr) + +define Build/Compile + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-host), + $(call Build/Compile/HostPyRunHost,,$(HOST_PYTHON_PIP_INSTALL_HOST) $(pkg)) + ) + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list), + $(call Build/Compile/HostPyRunTarget,,$(call HOST_PYTHON_PIP_INSTALL_TARGET,$(pkg)) $(pkg),$(CONFIG_PACKAGE_python-packages-envs)) + ) + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-cleanup), + $(call Build/Compile/HostPyRunTarget,,$(HOST_PYTHON_PIP_INSTALL_CLEANUP) $(pkg),$(CONFIG_PACKAGE_python-packages-envs)) + ) +endef + +define Package/python-packages/install + $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list), + $(CP) "$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg))"/* $(1) + ) + + find "$(PKG_INSTALL_DIR)/_cleanup" -mindepth 1 -depth | while read sf; do \ + tf="$$$${sf#$(PKG_INSTALL_DIR)/_cleanup/}"; \ + tf="$(1)/$$$$tf"; \ + if [ -f "$$$$tf" -o -L "$$$$tf" ]; then \ + rm -vf "$$$$tf"; \ + elif [ -d "$$$$tf" ]; then \ + rmdir -v -p "$$$$tf" || true; \ + fi \ + done +endef + +define Package/python-packages/extra_provides + echo $(CONFIG_PACKAGE_python-packages-extra-deps) | tr ' ' '\n' +endef + +$(eval $(call BuildPackage,python-packages)) diff --git a/lang/python-packages/README.md b/lang/python-packages/README.md new file mode 100644 index 000000000..3c09d8eb6 --- /dev/null +++ b/lang/python-packages/README.md @@ -0,0 +1,72 @@ +This package allows users to package python modules without creating package +Makefiles for each individual module and their dependencies. It provides a +way making packaging python packages faster and may also facilitate the process +of developing Makefiles for new python packages + +This is a raw DEVEL only package. Using it may entail a lot of implementation +details and you may need to resolve target dependencies and package details on +your own + +- Third party python packages may depend on features not included in e.g. + python-light +- Some python modules may require host install of another module to progress, + e.g. target cryptography requires host cffi +- Some python modules have external C library dependencies, e.g. pyOpenSSL + requires openssl libs +- Some packages may have an autoconf configure script whose arguments we + cannot control with pip and has to be passed on (hacked) by overriding some + environment variables + +## How it works + +1. Install host modules required for building target modules +2. Install each target module to separate directories +3. Install another copy of modules for cleanup purposes to make list of + installed files to be removed from target modules installed in step 2 + +Why should it be so + +1. Installing target cryptography requires host installation of cffi module +2. cryptography requires setuptools and pip will install its own copy with + --ignore-installed. When PACKAGE_python-setuptools is also selected, opkg + will complain of data file clashes if it was not removed here. + +Pip will handle dependency requirements of python modules, but external +dependencies like c libraries has to be prepared by the build system. The +issue is that there is currently no way to express such dependencies, thus may +cause build failure, e.g. pycrypto requires the presence of libgmp to build +successfully. + +## Tips + +If something goes wrong, we can add additional arguments to pip command +line to check the detailed build process. Some useful arguments may be + +- -v, for verbose output. Repeat this option if the current level of + verbosity is not enough +- --no-clean, for preserving pip build dir on build failure + +## Examples + +tornado (python-only module) + + CONFIG_PACKAGE_python-packages=y + CONFIG_PACKAGE_python-packages-list="tornado==4.4.2" + +cryptography (requires installation of host modules and cleanup on target modules) + + CONFIG_PACKAGE_python-packages=y + CONFIG_PACKAGE_python-packages-list-host="cffi" + CONFIG_PACKAGE_python-packages-list="cryptography" + CONFIG_PACKAGE_python-packages-list-cleanup="setuptools" + +pycrypto 2.7a1 (python module with autoconf configure script; depends on +libgmp; broken wmmintrin.h). 2.6.1 does not work because of a flaw in +the setup.py hardcoding host include directory + + CONFIG_PACKAGE_libgmp=y + CONFIG_PACKAGE_python-packages=y + CONFIG_PACKAGE_python-packages-list="https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz" + CONFIG_PACKAGE_python-packages-envs="ac_cv_header_wmmintrin_h=no build_alias=$(GNU_HOST_NAME) host_alias=$(GNU_TARGET_NAME) target_alias=$(GNU_TARGET_NAME)" + CONFIG_PACKAGE_python-packages-extra-deps="libgmp.so.10" +