LILiK
/
openwrt-packages-dist
5
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

Merge pull request #3352 from yousong/python-packages 

python-packages: initial version 1.0
lilik-openwrt-22.03
Yousong Zhou 8 years ago
committed by GitHub
parent
78d163bdea c6ddd3d7c0
commit
9f8dd657a1
5 changed files with 253 additions and 12 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +131
    -0
      lang/python-packages/Makefile
  2. +72
    -0
      lang/python-packages/README.md
  3. +18
    -1
      lang/python-pip/Makefile
  4. +16
    -5
      lang/python/files/python-host.mk
  5. +16
    -6
      lang/python/files/python-package.mk

+ 131
- 0
lang/python-packages/Makefile View File

@ -0,0 +1,131 @@
#
# Copyright (C) 2016 Yousong Zhou <yszhou4tech@gmail.com>
#
# This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
#
include $(TOPDIR)/rules.mk
PKG_NAME:=python-packages
PKG_VERSION:=1.0
PKG_RELEASE:=1
PKG_MAINTAINER:=Yousong Zhou <yszhou4tech@gmail.com>
#
# NOTE: move the host module installation to Host/Compile when
# HOST_CONFIG_DEPENDS is supported
#
# NOTE: PKG_CONFIG_DEPENDS cannot correctly track changes of string type config
# options, so you may want to do manual cleanup on config change.
#
PKG_CONFIG_DEPENDS:= \
CONFIG_PACKAGE_python-packages-list-host \
CONFIG_PACKAGE_python-packages-list \
CONFIG_PACKAGE_python-packages-list-cleanup \
CONFIG_PACKAGE_python-packages-envs \
CONFIG_PACKAGE_python-packages-extra-deps \
CONFIG_PACKAGE_python-packages-index-url \
CONFIG_PACKAGE_python-packages-pip-opts \
PKG_BUILD_DEPENDS:=python python-pip/host
include $(INCLUDE_DIR)/package.mk
$(call include_mk, python-host.mk)
$(call include_mk, python-package.mk)
define Package/python-packages
SUBMENU:=Python
SECTION:=lang
CATEGORY:=Languages
TITLE:=A dummy package for packaging python modules with pip
DEPENDS:=@DEVEL +python
endef
define Package/python-packages/config
if PACKAGE_python-packages
config PACKAGE_python-packages-list-host
string "List of python packages to install on host"
config PACKAGE_python-packages-list
string "List of python packages to install on target"
config PACKAGE_python-packages-list-cleanup
string "List of python packages to cleanup to avoid clash with existing packages"
config PACKAGE_python-packages-envs
string "Extra environment variables to pass on to pip and its children on target build"
config PACKAGE_python-packages-extra-deps
string "List of deps fulfilled but not tracked by the build system"
config PACKAGE_python-packages-index-url
string "Index URL passed to pip with --index-url"
config PACKAGE_python-packages-pip-opts
string "Additional arguments to pip command line"
endif
endef
CONFIG_PACKAGE_python-packages-list-host:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-host))
CONFIG_PACKAGE_python-packages-list:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list))
CONFIG_PACKAGE_python-packages-list-cleanup:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-cleanup))
CONFIG_PACKAGE_python-packages-envs:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-envs))
CONFIG_PACKAGE_python-packages-extra-deps:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-extra-deps))
CONFIG_PACKAGE_python-packages-pip-opts:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-pip-opts))
HOST_PYTHON_PIP:=$(STAGING_DIR)/host/bin/pip$(PYTHON_VERSION)
decr=$(word $(1),0 1 2 3 4 5 6 7 8 9 10)
recur=$(if $(subst 0,,$(2)),$(call recur,$(1),$(call decr,$(2)),$(call $(1)$(2),$(3))),$(3))
_req2dir1=$(subst >,gt,$(1))
_req2dir2=$(subst <,lt,$(1))
_req2dir3=$(subst >=,geq,$(1))
_req2dir4=$(subst <=,leq,$(1))
_req2dir5=$(subst ://,:::,$(1))
_req2dir6=$(subst *,_,$(1))
_req2dir7=$(subst ?,_,$(1))
req2dir=$(call recur,_req2dir,7,$(1))
# --ignore-installed, it may happen that host pip will ignore target install if
# it was already installed as host module, e.g. cffi deps of cryptograph
HOST_PYTHON_PIP_INSTALL=$(HOST_PYTHON_PIP) install \
--root=$(1) \
--prefix=$(2) \
--ignore-installed \
--no-compile \
$(if $(CONFIG_PACKAGE_python-packages-index-url), --index-url $(CONFIG_PACKAGE_python-packages-index-url)) \
$(if $(CONFIG_PACKAGE_python-packages-pip-opts), $(CONFIG_PACKAGE_python-packages-pip-opts)) \
HOST_PYTHON_PIP_INSTALL_HOST:=$(call HOST_PYTHON_PIP_INSTALL,$(STAGING_DIR)/host,"")
HOST_PYTHON_PIP_INSTALL_TARGET=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg)),/usr)
HOST_PYTHON_PIP_INSTALL_CLEANUP:=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/_cleanup,/usr)
define Build/Compile
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-host),
$(call Build/Compile/HostPyRunHost,,$(HOST_PYTHON_PIP_INSTALL_HOST) $(pkg))
)
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list),
$(call Build/Compile/HostPyRunTarget,,$(call HOST_PYTHON_PIP_INSTALL_TARGET,$(pkg)) $(pkg),$(CONFIG_PACKAGE_python-packages-envs))
)
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-cleanup),
$(call Build/Compile/HostPyRunTarget,,$(HOST_PYTHON_PIP_INSTALL_CLEANUP) $(pkg),$(CONFIG_PACKAGE_python-packages-envs))
)
endef
define Package/python-packages/install
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list),
$(CP) "$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg))"/* $(1)
)
find "$(PKG_INSTALL_DIR)/_cleanup" -mindepth 1 -depth | while read sf; do \
tf="$$$${sf#$(PKG_INSTALL_DIR)/_cleanup/}"; \
tf="$(1)/$$$$tf"; \
if [ -f "$$$$tf" -o -L "$$$$tf" ]; then \
rm -vf "$$$$tf"; \
elif [ -d "$$$$tf" ]; then \
rmdir -v -p "$$$$tf" || true; \
fi \
done
endef
define Package/python-packages/extra_provides
echo $(CONFIG_PACKAGE_python-packages-extra-deps) | tr ' ' '\n'
endef
$(eval $(call BuildPackage,python-packages))

+ 72
- 0
lang/python-packages/README.md View File

@ -0,0 +1,72 @@
This package allows users to package python modules without creating package
Makefiles for each individual module and their dependencies. It provides a
way making packaging python packages faster and may also facilitate the process
of developing Makefiles for new python packages
This is a raw DEVEL only package. Using it may entail a lot of implementation
details and you may need to resolve target dependencies and package details on
your own
- Third party python packages may depend on features not included in e.g.
python-light
- Some python modules may require host install of another module to progress,
e.g. target cryptography requires host cffi
- Some python modules have external C library dependencies, e.g. pyOpenSSL
requires openssl libs
- Some packages may have an autoconf configure script whose arguments we
cannot control with pip and has to be passed on (hacked) by overriding some
environment variables
## How it works
1. Install host modules required for building target modules
2. Install each target module to separate directories
3. Install another copy of modules for cleanup purposes to make list of
installed files to be removed from target modules installed in step 2
Why should it be so
1. Installing target cryptography requires host installation of cffi module
2. cryptography requires setuptools and pip will install its own copy with
--ignore-installed. When PACKAGE_python-setuptools is also selected, opkg
will complain of data file clashes if it was not removed here.
Pip will handle dependency requirements of python modules, but external
dependencies like c libraries has to be prepared by the build system. The
issue is that there is currently no way to express such dependencies, thus may
cause build failure, e.g. pycrypto requires the presence of libgmp to build
successfully.
## Tips
If something goes wrong, we can add additional arguments to pip command
line to check the detailed build process. Some useful arguments may be
- -v, for verbose output. Repeat this option if the current level of
verbosity is not enough
- --no-clean, for preserving pip build dir on build failure
## Examples
tornado (python-only module)
CONFIG_PACKAGE_python-packages=y
CONFIG_PACKAGE_python-packages-list="tornado==4.4.2"
cryptography (requires installation of host modules and cleanup on target modules)
CONFIG_PACKAGE_python-packages=y
CONFIG_PACKAGE_python-packages-list-host="cffi"
CONFIG_PACKAGE_python-packages-list="cryptography"
CONFIG_PACKAGE_python-packages-list-cleanup="setuptools"
pycrypto 2.7a1 (python module with autoconf configure script; depends on
libgmp; broken wmmintrin.h). 2.6.1 does not work because of a flaw in
the setup.py hardcoding host include directory
CONFIG_PACKAGE_libgmp=y
CONFIG_PACKAGE_python-packages=y
CONFIG_PACKAGE_python-packages-list="https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz"
CONFIG_PACKAGE_python-packages-envs="ac_cv_header_wmmintrin_h=no build_alias=$(GNU_HOST_NAME) host_alias=$(GNU_TARGET_NAME) target_alias=$(GNU_TARGET_NAME)"
CONFIG_PACKAGE_python-packages-extra-deps="libgmp.so.10"

+ 18
- 1
lang/python-pip/Makefile View File

@ -17,10 +17,16 @@ PKG_MD5SUM:=87083c0b9867963b29f7aba3613e8f4a
PKG_BUILD_DIR:=$(BUILD_DIR)/python-pip-$(PKG_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/python-pip-$(PKG_VERSION)
PKG_UNPACK=$(HOST_TAR) -C $(PKG_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE) PKG_UNPACK=$(HOST_TAR) -C $(PKG_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE)
HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/python-pip-$(PKG_VERSION)
HOST_UNPACK=$(HOST_TAR) -C $(HOST_BUILD_DIR) --strip-components=1 -xzf $(DL_DIR)/$(PKG_SOURCE)
PKG_USE_MIPS16:=0 PKG_USE_MIPS16:=0
HOST_BUILD_DEPENDS:=python python/host python-setuptools/host
include $(INCLUDE_DIR)/host-build.mk
include $(INCLUDE_DIR)/package.mk include $(INCLUDE_DIR)/package.mk
$(call include_mk, python-package.mk) $(call include_mk, python-package.mk)
$(call include_mk, python-host.mk)
define Package/python-pip define Package/python-pip
SUBMENU:=Python SUBMENU:=Python
@ -54,6 +60,17 @@ define PyPackage/python-pip/install
$(INSTALL_CONF) ./files/pip.conf $(1)/etc/ $(INSTALL_CONF) ./files/pip.conf $(1)/etc/
endef endef
define Host/Compile
$(call Build/Compile/HostPyMod,,\
install --root="$(STAGING_DIR)/host" --prefix="" \
--single-version-externally-managed \
)
endef
define Host/Install
endef
$(eval $(call HostBuild))
$(eval $(call PyPackage,python-pip)) $(eval $(call PyPackage,python-pip))
$(eval $(call BuildPackage,python-pip)) $(eval $(call BuildPackage,python-pip))

+ 16
- 5
lang/python/files/python-host.mk View File

@ -31,12 +31,12 @@ define HostPython
$(HOST_PYTHON_BIN) $(2); $(HOST_PYTHON_BIN) $(2);
endef endef
# $(1) => build subdir
# $(2) => additional arguments to setup.py
# $(1) => commands to execute before running pythons script
# $(2) => python script and its arguments
# $(3) => additional variables # $(3) => additional variables
define Build/Compile/HostPyMod
define Build/Compile/HostPyRunHost
$(call HostPython, \ $(call HostPython, \
cd $(HOST_BUILD_DIR)/$(strip $(1)); \
$(if $(1),$(1);) \
CC="$(HOSTCC)" \ CC="$(HOSTCC)" \
CCSHARED="$(HOSTCC) $(HOST_FPIC)" \ CCSHARED="$(HOSTCC) $(HOST_FPIC)" \
CXX="$(HOSTCXX)" \ CXX="$(HOSTCXX)" \
@ -48,9 +48,20 @@ define Build/Compile/HostPyMod
_PYTHON_HOST_PLATFORM=linux2 \ _PYTHON_HOST_PLATFORM=linux2 \
$(3) \ $(3) \
, \ , \
./setup.py $(2) \
$(2) \
, \ , \
HOST \ HOST \
) )
endef endef
# $(1) => build subdir
# $(2) => additional arguments to setup.py
# $(3) => additional variables
define Build/Compile/HostPyMod
$(call Build/Compile/HostPyRunHost, \
cd $(HOST_BUILD_DIR)/$(strip $(1)), \
./setup.py $(2), \
$(3))
endef

+ 16
- 6
lang/python/files/python-package.mk View File

@ -82,13 +82,12 @@ endef
$(call include_mk, python-host.mk) $(call include_mk, python-host.mk)
# $(1) => build subdir
# $(2) => additional arguments to setup.py
# $(1) => commands to execute before running pythons script
# $(2) => python script and its arguments
# $(3) => additional variables # $(3) => additional variables
define Build/Compile/PyMod
$(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON_PKG_DIR)
define Build/Compile/HostPyRunTarget
$(call HostPython, \ $(call HostPython, \
cd $(PKG_BUILD_DIR)/$(strip $(1)); \
$(if $(1),$(1);) \
CC="$(TARGET_CC)" \ CC="$(TARGET_CC)" \
CCSHARED="$(TARGET_CC) $(FPIC)" \ CCSHARED="$(TARGET_CC) $(FPIC)" \
CXX="$(TARGET_CXX)" \ CXX="$(TARGET_CXX)" \
@ -101,8 +100,19 @@ define Build/Compile/PyMod
__PYVENV_LAUNCHER__="/usr/bin/$(PYTHON)" \ __PYVENV_LAUNCHER__="/usr/bin/$(PYTHON)" \
$(3) \ $(3) \
, \ , \
./setup.py $(2) \
$(2) \
) )
endef
# $(1) => build subdir
# $(2) => additional arguments to setup.py
# $(3) => additional variables
define Build/Compile/PyMod
$(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON_PKG_DIR)
$(call Build/Compile/HostPyRunTarget, \
cd $(PKG_BUILD_DIR)/$(strip $(1)), \
./setup.py $(2), \
$(3))
find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
endef endef


Write Preview
Loading…
Cancel
Save