Browse Source

Merge pull request #13131 from jefferyto/python-pip-hash-checking

python3: Use hash-checking mode when installing host pip packages
lilik-openwrt-22.03
Rosen Penev 4 years ago
committed by GitHub
parent
commit
bf3b6af205
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 65 additions and 18 deletions
  1. +37
    -5
      lang/python/README.md
  2. +1
    -1
      lang/python/bcrypt/Makefile
  3. +1
    -0
      lang/python/host-pip-requirements/Cython.txt
  4. +2
    -0
      lang/python/host-pip-requirements/Django-1.11.txt
  5. +2
    -0
      lang/python/host-pip-requirements/cffi.txt
  6. +1
    -0
      lang/python/host-pip-requirements/ply.txt
  7. +1
    -0
      lang/python/host-pip-requirements/setuptools-scm.txt
  8. +1
    -0
      lang/python/host-pip-requirements/toml.txt
  9. +1
    -1
      lang/python/numpy/Makefile
  10. +1
    -1
      lang/python/python-cryptography/Makefile
  11. +1
    -1
      lang/python/python-jsonschema/Makefile
  12. +2
    -2
      lang/python/python-pycparser/Makefile
  13. +1
    -1
      lang/python/python-pynacl/Makefile
  14. +1
    -1
      lang/python/python-zipp/Makefile
  15. +2
    -1
      lang/python/python3-host.mk
  16. +8
    -2
      lang/python/python3-package.mk
  17. +2
    -2
      net/seafile-seahub/Makefile

+ 37
- 5
lang/python/README.md View File

@ -330,13 +330,45 @@ endef
These can be installed via pip and ideally they should only be installed like this, because it's a bit simpler than running them through the OpenWrt build system.
Which is why [for example] if you need python cffi on the host build, it's easier to just add it via:
#### Requirements files
All host-side Python packages are installed with pip using [requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files), with [hash-checking mode](https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode) enabled. These requirements files are stored in the [host-pip-requirements](./host-pip-requirements) directory.
Each requirements file is named after the Python package it installs and contains the package's pinned version and `--hash` option. The `--hash` option value is the SHA256 hash of the package's source tarball; this value can be found on [pypi.org](https://pypi.org/).
For example, the requirements file for setuptools-scm ([setuptools-scm.txt](./host-pip-requirements/setuptools-scm.txt)) contains:
```
setuptools-scm==4.1.2 --hash=sha256:a8994582e716ec690f33fec70cca0f85bd23ec974e3f783233e4879090a7faa8
```
If the Python package to be installed depends on other Python packages, those dependencies, with their pinned versions and `--hash` options, also need to be specified in the requirements file. For instance, [cffi.txt](./host-pip-requirements/cffi.txt) includes information for pycparser because pycparser is a dependency of cffi and will be installed with cffi.
There are two types of requirements files in [host-pip-requirements](./host-pip-requirements):
* Installs the latest version of a Python package.
A requirements file of this type is named with the package name only (for example, [setuptools-scm.txt](./host-pip-requirements/setuptools-scm.txt)) and is used when there is no strict version requirement.
These files will be updated as newer versions of the Python packages are available.
* Installs a specific version of a Python package.
A requirements file of this type is named with the package name and version number (for example, [Django-1.11.txt](./host-pip-requirements/Django-1.11.txt)) and is used when a specific (usually older) version is required.
Installing the latest versions of packages is preferred over specific versions whenever possible.
#### Installing host-side Python packages
Set `HOST_PYTHON3_PACKAGE_BUILD_DEPENDS` to the names of one or more requirements files in [host-pip-requirements](./host-pip-requirements), without the directory path or ".txt" extension.
For example:
```
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi==$(PKG_VERSION)"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm
```
[cffi is one of those packages that needs a host-side package installed].
This works reasonably well in the current OpenWrt build system, as binaries get built for this package and get installed in the staging-dir `$(STAGING_DIR)/usr/lib/pythonX.Y/site-packages`.
The Python package will be installed in `$(STAGING_DIR_HOSTPKG)/lib/pythonX.Y/site-packages`.
`PKG_BUILD_PARALLEL:=0` is necessary because installing packages with multiple concurrent pip processes can lead to [errors or unexpected results](https://github.com/pypa/pip/issues/2361).
Parallel builds need to be disabled because installing packages with multiple concurrent pip processes can lead to [errors or unexpected results](https://github.com/pypa/pip/issues/2361).

+ 1
- 1
lang/python/bcrypt/Makefile View File

@ -17,7 +17,7 @@ PKG_LICENSE_FILES:=LICENSE
PKG_BUILD_DEPENDS:=libffi/host
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.1"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi # cffi>=1.1
include ../pypi.mk
include $(INCLUDE_DIR)/package.mk


+ 1
- 0
lang/python/host-pip-requirements/Cython.txt View File

@ -0,0 +1 @@
Cython==0.29.21 --hash=sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad

+ 2
- 0
lang/python/host-pip-requirements/Django-1.11.txt View File

@ -0,0 +1,2 @@
Django==1.11.29 --hash=sha256:4200aefb6678019a0acf0005cd14cfce3a5e6b9b90d06145fcdd2e474ad4329c
pytz==2020.1 --hash=sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048

+ 2
- 0
lang/python/host-pip-requirements/cffi.txt View File

@ -0,0 +1,2 @@
cffi==1.14.2 --hash=sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b
pycparser==2.20 --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0

+ 1
- 0
lang/python/host-pip-requirements/ply.txt View File

@ -0,0 +1 @@
ply==3.11 --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3

+ 1
- 0
lang/python/host-pip-requirements/setuptools-scm.txt View File

@ -0,0 +1 @@
setuptools-scm==4.1.2 --hash=sha256:a8994582e716ec690f33fec70cca0f85bd23ec974e3f783233e4879090a7faa8

+ 1
- 0
lang/python/host-pip-requirements/toml.txt View File

@ -0,0 +1 @@
toml==0.10.1 --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f

+ 1
- 1
lang/python/numpy/Makefile View File

@ -22,7 +22,7 @@ PKG_CPE_ID:=cpe:/a:numpy:numpy
# yes, zip... sigh
PYPI_SOURCE_EXT:=zip
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="Cython==0.29.21"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=Cython # Cython>=0.29.21
include ../pypi.mk
include $(INCLUDE_DIR)/package.mk


+ 1
- 1
lang/python/python-cryptography/Makefile View File

@ -21,7 +21,7 @@ PKG_MAINTAINER:=Jeffery To <jeffery.to@gmail.com>, Alexandru Ardelean <ardeleana
PKG_BUILD_DEPENDS:=libffi/host
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.8,!=1.11.3"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi # cffi>=1.8,!=1.11.3
include ../pypi.mk
include $(INCLUDE_DIR)/package.mk


+ 1
- 1
lang/python/python-jsonschema/Makefile View File

@ -12,7 +12,7 @@ PKG_LICENSE:=MIT
PKG_LICENSE_FILES:=COPYING
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools_scm
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm
include ../pypi.mk
include $(INCLUDE_DIR)/package.mk


+ 2
- 2
lang/python/python-pycparser/Makefile View File

@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
PKG_NAME:=python-pycparser
PKG_VERSION:=2.20
PKG_RELEASE:=3
PKG_RELEASE:=4
PYPI_NAME:=pycparser
PKG_HASH:=2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0
@ -19,7 +19,7 @@ PKG_LICENSE_FILES:=LICENSE
PKG_MAINTAINER:=Jeffery To <jeffery.to@gmail.com>
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="ply==3.10"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=ply # ply==3.10
include ../pypi.mk
include $(INCLUDE_DIR)/package.mk


+ 1
- 1
lang/python/python-pynacl/Makefile View File

@ -14,7 +14,7 @@ PKG_LICENSE_FILES:=LICENSE
PKG_BUILD_DEPENDS:=libffi/host
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.4.1"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi # cffi>=1.4.1
PYTHON3_PKG_SETUP_VARS:= SODIUM_INSTALL=system


+ 1
- 1
lang/python/python-zipp/Makefile View File

@ -12,7 +12,7 @@ PKG_LICENSE:=MIT
PKG_LICENSE_FILES:=LICENSE
PKG_BUILD_PARALLEL:=0
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="setuptools_scm[toml] >= 3.4.1"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm toml # setuptools_scm[toml] >= 3.4.1
include ../pypi.mk
include $(INCLUDE_DIR)/package.mk


+ 2
- 1
lang/python/python3-host.mk View File

@ -55,10 +55,11 @@ HOST_PYTHON3_PIP_CACHE_DIR:=$(DL_DIR)/pip-cache
define HostPython3/PipInstall
$(HOST_PYTHON3_VARS) \
$(HOST_PYTHON3_PIP) \
--disable-pip-version-check \
--cache-dir "$(HOST_PYTHON3_PIP_CACHE_DIR)" \
--disable-pip-version-check \
install \
--no-binary :all: \
--require-hashes \
$(1)
ifdef CONFIG_PYTHON3_HOST_PIP_CACHE_WORLD_READABLE
$(FIND) $(HOST_PYTHON3_PIP_CACHE_DIR) -not -type d -exec chmod go+r '{}' \;


+ 8
- 2
lang/python/python3-package.mk View File

@ -194,13 +194,19 @@ PYTHON3_PKG_SETUP_GLOBAL_ARGS ?=
PYTHON3_PKG_SETUP_ARGS ?= --single-version-externally-managed
PYTHON3_PKG_SETUP_VARS ?=
PYTHON3_PKG_HOST_PIP_INSTALL_ARGS = \
$(foreach req,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS), \
--requirement \
$(if $(findstring /,$(req)),$(req),$(python3_mk_path)host-pip-requirements/$(req).txt) \
)
define Py3Build/FindStdlibDepends
$(SHELL) $(python3_mk_path)python3-find-stdlib-depends.sh -n "$(PKG_NAME)" "$(PKG_BUILD_DIR)"
endef
define Py3Build/Compile/Default
$(if $(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS),
$(call HostPython3/PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS))
$(if $(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS), \
$(call HostPython3/PipInstall,$(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS)) \
)
$(call Python3/ModSetup, \
$(PYTHON3_PKG_SETUP_DIR), \


+ 2
- 2
net/seafile-seahub/Makefile View File

@ -21,7 +21,7 @@ PKG_LICENSE_FILES:=LICENSE.txt
PKG_BUILD_DIR:=$(BUILD_DIR)/seahub-$(PKG_VERSION)-server
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="Django~=1.11"
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=Django-1.11
PKG_BUILD_PARALLEL:=0
PYTHON3_PKG_BUILD:=0
@ -76,7 +76,7 @@ MAKE_VARS += \
DJANGO_ADMIN_PY="$(STAGING_DIR_HOSTPKG)/bin/django-admin"
define Build/Compile
$(call HostPython3/PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS))
$(call HostPython3/PipInstall,$(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS))
$(call Build/Compile/Default,locale)
endef


Loading…
Cancel
Save