Browse Source
Merge pull request #3352 from yousong/python-packages
python-packages: initial version 1.0lilik-openwrt-22.03
Yousong Zhou
8 years ago
committed by
GitHub
GitHub
5 changed files with 253 additions and 12 deletions
Split View
Diff Options
-
+131 -0lang/python-packages/Makefile
-
+72 -0lang/python-packages/README.md
-
+18 -1lang/python-pip/Makefile
-
+16 -5lang/python/files/python-host.mk
-
+16 -6lang/python/files/python-package.mk
+ 131
- 0
lang/python-packages/Makefile
View File
| @ -0,0 +1,131 @@ | |||
| # | |||
| # Copyright (C) 2016 Yousong Zhou <yszhou4tech@gmail.com> | |||
| # | |||
| # This is free software, licensed under the GNU General Public License v2. | |||
| # See /LICENSE for more information. | |||
| # | |||
| include $(TOPDIR)/rules.mk | |||
| PKG_NAME:=python-packages | |||
| PKG_VERSION:=1.0 | |||
| PKG_RELEASE:=1 | |||
| PKG_MAINTAINER:=Yousong Zhou <yszhou4tech@gmail.com> | |||
| # | |||
| # NOTE: move the host module installation to Host/Compile when | |||
| # HOST_CONFIG_DEPENDS is supported | |||
| # | |||
| # NOTE: PKG_CONFIG_DEPENDS cannot correctly track changes of string type config | |||
| # options, so you may want to do manual cleanup on config change. | |||
| # | |||
| PKG_CONFIG_DEPENDS:= \ | |||
| CONFIG_PACKAGE_python-packages-list-host \ | |||
| CONFIG_PACKAGE_python-packages-list \ | |||
| CONFIG_PACKAGE_python-packages-list-cleanup \ | |||
| CONFIG_PACKAGE_python-packages-envs \ | |||
| CONFIG_PACKAGE_python-packages-extra-deps \ | |||
| CONFIG_PACKAGE_python-packages-index-url \ | |||
| CONFIG_PACKAGE_python-packages-pip-opts \ | |||
| PKG_BUILD_DEPENDS:=python python-pip/host | |||
| include $(INCLUDE_DIR)/package.mk | |||
| $(call include_mk, python-host.mk) | |||
| $(call include_mk, python-package.mk) | |||
| define Package/python-packages | |||
| SUBMENU:=Python | |||
| SECTION:=lang | |||
| CATEGORY:=Languages | |||
| TITLE:=A dummy package for packaging python modules with pip | |||
| DEPENDS:=@DEVEL +python | |||
| endef | |||
| define Package/python-packages/config | |||
| if PACKAGE_python-packages | |||
| config PACKAGE_python-packages-list-host | |||
| string "List of python packages to install on host" | |||
| config PACKAGE_python-packages-list | |||
| string "List of python packages to install on target" | |||
| config PACKAGE_python-packages-list-cleanup | |||
| string "List of python packages to cleanup to avoid clash with existing packages" | |||
| config PACKAGE_python-packages-envs | |||
| string "Extra environment variables to pass on to pip and its children on target build" | |||
| config PACKAGE_python-packages-extra-deps | |||
| string "List of deps fulfilled but not tracked by the build system" | |||
| config PACKAGE_python-packages-index-url | |||
| string "Index URL passed to pip with --index-url" | |||
| config PACKAGE_python-packages-pip-opts | |||
| string "Additional arguments to pip command line" | |||
| endif | |||
| endef | |||
| CONFIG_PACKAGE_python-packages-list-host:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-host)) | |||
| CONFIG_PACKAGE_python-packages-list:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list)) | |||
| CONFIG_PACKAGE_python-packages-list-cleanup:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-cleanup)) | |||
| CONFIG_PACKAGE_python-packages-envs:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-envs)) | |||
| CONFIG_PACKAGE_python-packages-extra-deps:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-extra-deps)) | |||
| CONFIG_PACKAGE_python-packages-pip-opts:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-pip-opts)) | |||
| HOST_PYTHON_PIP:=$(STAGING_DIR)/host/bin/pip$(PYTHON_VERSION) | |||
| decr=$(word $(1),0 1 2 3 4 5 6 7 8 9 10) | |||
| recur=$(if $(subst 0,,$(2)),$(call recur,$(1),$(call decr,$(2)),$(call $(1)$(2),$(3))),$(3)) | |||
| _req2dir1=$(subst >,gt,$(1)) | |||
| _req2dir2=$(subst <,lt,$(1)) | |||
| _req2dir3=$(subst >=,geq,$(1)) | |||
| _req2dir4=$(subst <=,leq,$(1)) | |||
| _req2dir5=$(subst ://,:::,$(1)) | |||
| _req2dir6=$(subst *,_,$(1)) | |||
| _req2dir7=$(subst ?,_,$(1)) | |||
| req2dir=$(call recur,_req2dir,7,$(1)) | |||
| # --ignore-installed, it may happen that host pip will ignore target install if | |||
| # it was already installed as host module, e.g. cffi deps of cryptograph | |||
| HOST_PYTHON_PIP_INSTALL=$(HOST_PYTHON_PIP) install \ | |||
| --root=$(1) \ | |||
| --prefix=$(2) \ | |||
| --ignore-installed \ | |||
| --no-compile \ | |||
| $(if $(CONFIG_PACKAGE_python-packages-index-url), --index-url $(CONFIG_PACKAGE_python-packages-index-url)) \ | |||
| $(if $(CONFIG_PACKAGE_python-packages-pip-opts), $(CONFIG_PACKAGE_python-packages-pip-opts)) \ | |||
| HOST_PYTHON_PIP_INSTALL_HOST:=$(call HOST_PYTHON_PIP_INSTALL,$(STAGING_DIR)/host,"") | |||
| HOST_PYTHON_PIP_INSTALL_TARGET=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg)),/usr) | |||
| HOST_PYTHON_PIP_INSTALL_CLEANUP:=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/_cleanup,/usr) | |||
| define Build/Compile | |||
| $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-host), | |||
| $(call Build/Compile/HostPyRunHost,,$(HOST_PYTHON_PIP_INSTALL_HOST) $(pkg)) | |||
| ) | |||
| $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list), | |||
| $(call Build/Compile/HostPyRunTarget,,$(call HOST_PYTHON_PIP_INSTALL_TARGET,$(pkg)) $(pkg),$(CONFIG_PACKAGE_python-packages-envs)) | |||
| ) | |||
| $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-cleanup), | |||
| $(call Build/Compile/HostPyRunTarget,,$(HOST_PYTHON_PIP_INSTALL_CLEANUP) $(pkg),$(CONFIG_PACKAGE_python-packages-envs)) | |||
| ) | |||
| endef | |||
| define Package/python-packages/install | |||
| $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list), | |||
| $(CP) "$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg))"/* $(1) | |||
| ) | |||
| find "$(PKG_INSTALL_DIR)/_cleanup" -mindepth 1 -depth | while read sf; do \ | |||
| tf="$$$${sf#$(PKG_INSTALL_DIR)/_cleanup/}"; \ | |||
| tf="$(1)/$$$$tf"; \ | |||
| if [ -f "$$$$tf" -o -L "$$$$tf" ]; then \ | |||
| rm -vf "$$$$tf"; \ | |||
| elif [ -d "$$$$tf" ]; then \ | |||
| rmdir -v -p "$$$$tf" || true; \ | |||
| fi \ | |||
| done | |||
| endef | |||
| define Package/python-packages/extra_provides | |||
| echo $(CONFIG_PACKAGE_python-packages-extra-deps) | tr ' ' '\n' | |||
| endef | |||
| $(eval $(call BuildPackage,python-packages)) | |||
+ 72
- 0
lang/python-packages/README.md
View File
| @ -0,0 +1,72 @@ | |||
| This package allows users to package python modules without creating package | |||
| Makefiles for each individual module and their dependencies. It provides a | |||
| way making packaging python packages faster and may also facilitate the process | |||
| of developing Makefiles for new python packages | |||
| This is a raw DEVEL only package. Using it may entail a lot of implementation | |||
| details and you may need to resolve target dependencies and package details on | |||
| your own | |||
| - Third party python packages may depend on features not included in e.g. | |||
| python-light | |||
| - Some python modules may require host install of another module to progress, | |||
| e.g. target cryptography requires host cffi | |||
| - Some python modules have external C library dependencies, e.g. pyOpenSSL | |||
| requires openssl libs | |||
| - Some packages may have an autoconf configure script whose arguments we | |||
| cannot control with pip and has to be passed on (hacked) by overriding some | |||
| environment variables | |||
| ## How it works | |||
| 1. Install host modules required for building target modules | |||
| 2. Install each target module to separate directories | |||
| 3. Install another copy of modules for cleanup purposes to make list of | |||
| installed files to be removed from target modules installed in step 2 | |||
| Why should it be so | |||
| 1. Installing target cryptography requires host installation of cffi module | |||
| 2. cryptography requires setuptools and pip will install its own copy with | |||
| --ignore-installed. When PACKAGE_python-setuptools is also selected, opkg | |||
| will complain of data file clashes if it was not removed here. | |||
| Pip will handle dependency requirements of python modules, but external | |||
| dependencies like c libraries has to be prepared by the build system. The | |||
| issue is that there is currently no way to express such dependencies, thus may | |||
| cause build failure, e.g. pycrypto requires the presence of libgmp to build | |||
| successfully. | |||
| ## Tips | |||
| If something goes wrong, we can add additional arguments to pip command | |||
| line to check the detailed build process. Some useful arguments may be | |||
| - -v, for verbose output. Repeat this option if the current level of | |||
| verbosity is not enough | |||
| - --no-clean, for preserving pip build dir on build failure | |||
| ## Examples | |||
| tornado (python-only module) | |||
| CONFIG_PACKAGE_python-packages=y | |||
| CONFIG_PACKAGE_python-packages-list="tornado==4.4.2" | |||
| cryptography (requires installation of host modules and cleanup on target modules) | |||
| CONFIG_PACKAGE_python-packages=y | |||
| CONFIG_PACKAGE_python-packages-list-host="cffi" | |||
| CONFIG_PACKAGE_python-packages-list="cryptography" | |||
| CONFIG_PACKAGE_python-packages-list-cleanup="setuptools" | |||
| pycrypto 2.7a1 (python module with autoconf configure script; depends on | |||
| libgmp; broken wmmintrin.h). 2.6.1 does not work because of a flaw in | |||
| the setup.py hardcoding host include directory | |||
| CONFIG_PACKAGE_libgmp=y | |||
| CONFIG_PACKAGE_python-packages=y | |||
| CONFIG_PACKAGE_python-packages-list="https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz" | |||
| CONFIG_PACKAGE_python-packages-envs="ac_cv_header_wmmintrin_h=no build_alias=$(GNU_HOST_NAME) host_alias=$(GNU_TARGET_NAME) target_alias=$(GNU_TARGET_NAME)" | |||
| CONFIG_PACKAGE_python-packages-extra-deps="libgmp.so.10" | |||