You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

242 lines
6.8 KiB

python3: Use default _PYTHON_HOST_PLATFORM This lets the Python build process set _PYTHON_HOST_PLATFORM instead of forcing an explicit value. Also: * Save the target _PYTHON_HOST_PLATFORM value during Build/InstallDev for use when building target Python packages (in python3-package.mk). * Use the (mostly) default PYTHON_FOR_BUILD value, instead patch configure to remove the platform triplet from the sysconfigdata file name. * Remove the "CROSS_COMPILE=yes" make variable (there is no indication that this variable is necessary). * Force host pip to build packages from source instead of downloading binary wheels. Previously, host pip can download universal (platform-independent) wheels but not platform-specific wheels, because of the custom _PYTHON_HOST_PLATFORM value. (Packages that do not have universal wheels would be compiled from source.) With a correct _PYTHON_HOST_PLATFORM, host pip can install platform-specific wheels as well. However, the pre-built shared object (.so) files in these wheels will have the host's platform triplet in their file names. When target Python packages are built (using the target's _PYTHON_HOST_PLATFORM), Python will not use these shared object files. By forcing host pip to build packages from source, the built shared object files will not have the platform triplet in their file names. (Host Python has been patched to remove the platform triplet from file names.) This allows these packages to be used when building target Python packages. (The net effect of this complete change is that platform-dependent packages will continue to be compiled from source, while platform-independent packages will now also be compiled from source.) Signed-off-by: Jeffery To <jeffery.to@gmail.com>
4 years ago
python3: Use default _PYTHON_HOST_PLATFORM This lets the Python build process set _PYTHON_HOST_PLATFORM instead of forcing an explicit value. Also: * Save the target _PYTHON_HOST_PLATFORM value during Build/InstallDev for use when building target Python packages (in python3-package.mk). * Use the (mostly) default PYTHON_FOR_BUILD value, instead patch configure to remove the platform triplet from the sysconfigdata file name. * Remove the "CROSS_COMPILE=yes" make variable (there is no indication that this variable is necessary). * Force host pip to build packages from source instead of downloading binary wheels. Previously, host pip can download universal (platform-independent) wheels but not platform-specific wheels, because of the custom _PYTHON_HOST_PLATFORM value. (Packages that do not have universal wheels would be compiled from source.) With a correct _PYTHON_HOST_PLATFORM, host pip can install platform-specific wheels as well. However, the pre-built shared object (.so) files in these wheels will have the host's platform triplet in their file names. When target Python packages are built (using the target's _PYTHON_HOST_PLATFORM), Python will not use these shared object files. By forcing host pip to build packages from source, the built shared object files will not have the platform triplet in their file names. (Host Python has been patched to remove the platform triplet from file names.) This allows these packages to be used when building target Python packages. (The net effect of this complete change is that platform-dependent packages will continue to be compiled from source, while platform-independent packages will now also be compiled from source.) Signed-off-by: Jeffery To <jeffery.to@gmail.com>
4 years ago
  1. #
  2. # Copyright (C) 2007-2016 OpenWrt.org
  3. #
  4. # This is free software, licensed under the GNU General Public License v2.
  5. # See /LICENSE for more information.
  6. #
  7. # Note: include this file after `include $(TOPDIR)/rules.mk in your package Makefile
  8. python3_mk_path:=$(dir $(lastword $(MAKEFILE_LIST)))
  9. include $(python3_mk_path)python3-host.mk
  10. PYTHON3_DIR:=$(STAGING_DIR)/usr
  11. PYTHON3_INC_DIR:=$(PYTHON3_DIR)/include/python$(PYTHON3_VERSION)
  12. PYTHON3_LIB_DIR:=$(PYTHON3_DIR)/lib/python$(PYTHON3_VERSION)
  13. PYTHON3_PKG_DIR:=/usr/lib/python$(PYTHON3_VERSION)/site-packages
  14. PYTHON3:=python$(PYTHON3_VERSION)
  15. PYTHON3PATH:=$(PYTHON3_LIB_DIR):$(STAGING_DIR)/$(PYTHON3_PKG_DIR):$(PKG_INSTALL_DIR)/$(PYTHON3_PKG_DIR)
  16. -include $(PYTHON3_LIB_DIR)/config-$(PYTHON3_VERSION)/Makefile-vars
  17. # These configure args are needed in detection of path to Python header files
  18. # using autotools.
  19. CONFIGURE_ARGS += \
  20. _python_sysroot="$(STAGING_DIR)" \
  21. _python_prefix="/usr" \
  22. _python_exec_prefix="/usr"
  23. PYTHON3_VARS = \
  24. CC="$(TARGET_CC)" \
  25. CCSHARED="$(TARGET_CC) $(FPIC)" \
  26. CXX="$(TARGET_CXX)" \
  27. LD="$(TARGET_CC)" \
  28. LDSHARED="$(TARGET_CC) -shared" \
  29. CFLAGS="$(TARGET_CFLAGS)" \
  30. CPPFLAGS="$(TARGET_CPPFLAGS) -I$(PYTHON3_INC_DIR)" \
  31. LDFLAGS="$(TARGET_LDFLAGS) -lpython$(PYTHON3_VERSION)" \
  32. _PYTHON_HOST_PLATFORM="$(_PYTHON_HOST_PLATFORM)" \
  33. __PYVENV_LAUNCHER__="/usr/bin/$(PYTHON3)" \
  34. PYTHONPATH="$(PYTHON3PATH)" \
  35. PYTHONDONTWRITEBYTECODE=1 \
  36. PYTHONOPTIMIZE="" \
  37. _python_sysroot="$(STAGING_DIR)" \
  38. _python_prefix="/usr" \
  39. _python_exec_prefix="/usr"
  40. # $(1) => directory of python script
  41. # $(2) => python script and its arguments
  42. # $(3) => additional variables
  43. define Python3/Run
  44. cd "$(if $(strip $(1)),$(strip $(1)),.)" && \
  45. $(PYTHON3_VARS) \
  46. $(3) \
  47. $(HOST_PYTHON3_BIN) $(2)
  48. endef
  49. # $(1) => build subdir
  50. # $(2) => additional arguments to setup.py
  51. # $(3) => additional variables
  52. define Python3/ModSetup
  53. $(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON3_PKG_DIR)
  54. $(call Python3/Run, \
  55. $(PKG_BUILD_DIR)/$(strip $(1)), \
  56. setup.py $(2), \
  57. $(3))
  58. endef
  59. define Python3/FixShebang
  60. $(SED) "1"'!'"b;s,^#"'!'".*python.*,#"'!'"/usr/bin/python3," -i --follow-symlinks $(1)
  61. endef
  62. # default max recursion is 10
  63. PYTHON3_COMPILEALL_MAX_RECURSION_LEVEL:=20
  64. # $(1) => directory of python source files to compile
  65. #
  66. # XXX [So that you won't goof as I did]
  67. # Note: Yes, I tried to use the -O & -OO flags here.
  68. # However the generated byte-codes were not portable.
  69. # So, we just stuck to un-optimized byte-codes,
  70. # which is still way better/faster than running
  71. # Python sources all the time.
  72. #
  73. # Setting a fixed hash seed value is less secure than using
  74. # random seed values, but is necessary for reproducible builds
  75. # (for now).
  76. #
  77. # Should revisit this when https://bugs.python.org/issue37596
  78. # (and other related reproducibility issues) are fixed.
  79. define Python3/CompileAll
  80. $(call Python3/Run,, \
  81. -m compileall -r "$(PYTHON3_COMPILEALL_MAX_RECURSION_LEVEL)" -b -d '/' $(1),
  82. $(if $(SOURCE_DATE_EPOCH),PYTHONHASHSEED="$(SOURCE_DATE_EPOCH)")
  83. )
  84. endef
  85. # $(1) => target directory
  86. define Python3/DeleteSourceFiles
  87. $(FIND) $(1) -type f -name '*.py' -delete
  88. endef
  89. # $(1) => target directory
  90. define Python3/DeleteNonSourceFiles
  91. $(FIND) $(1) -not -type d -not -name '*.py' -delete
  92. endef
  93. # $(1) => target directory
  94. define Python3/DeleteEmptyDirs
  95. $(FIND) $(1) -mindepth 1 -empty -type d -not -path '$(1)/CONTROL' -not -path '$(1)/CONTROL/*' -delete
  96. endef
  97. # Py3Package
  98. define Py3Package/filespec/Default
  99. +|$(PYTHON3_PKG_DIR)
  100. endef
  101. # $(1) => package name
  102. # $(2) => src directory
  103. # $(3) => dest directory
  104. define Py3Package/ProcessFilespec
  105. $(eval $(call shexport,Py3Package/$(1)/filespec))
  106. $(SHELL) $(python3_mk_path)python-package-install.sh \
  107. "$(2)" "$(3)" "$$$$$(call shvar,Py3Package/$(1)/filespec)"
  108. endef
  109. define Py3Package
  110. define Package/$(1)-src
  111. $(call Package/$(1))
  112. DEPENDS:=
  113. CONFLICTS:=
  114. PROVIDES:=
  115. EXTRA_DEPENDS:=
  116. TITLE+= (sources)
  117. USERID:=
  118. MENU:=
  119. endef
  120. define Package/$(1)-src/description
  121. $$(call Package/$(1)/description)
  122. This package contains the Python source files for $(1).
  123. endef
  124. define Package/$(1)-src/config
  125. depends on PACKAGE_$(1)
  126. endef
  127. # Add default PyPackage filespec none defined
  128. ifeq ($(origin Py3Package/$(1)/filespec),undefined)
  129. Py3Package/$(1)/filespec=$$(Py3Package/filespec/Default)
  130. endif
  131. ifndef Py3Package/$(1)/install
  132. define Py3Package/$(1)/install
  133. if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \
  134. $(INSTALL_DIR) $$(1)/usr/bin ; \
  135. $(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \
  136. fi
  137. endef
  138. endif
  139. ifndef Package/$(1)/install
  140. define Package/$(1)/install
  141. $$(call Py3Package/$(1)/install,$$(1))
  142. $$(call Py3Package/ProcessFilespec,$(1),$(PKG_INSTALL_DIR),$$(1))
  143. $(FIND) $$(1) -name '*.exe' -delete
  144. $$(call Python3/CompileAll,$$(1))
  145. $$(call Python3/DeleteSourceFiles,$$(1))
  146. $$(call Python3/DeleteEmptyDirs,$$(1))
  147. if [ -d "$$(1)/usr/bin" ]; then \
  148. $$(call Python3/FixShebang,$$(1)/usr/bin/*) ; \
  149. fi
  150. endef
  151. define Package/$(1)-src/install
  152. $$(call Py3Package/$(1)/install,$$(1))
  153. $$(call Py3Package/ProcessFilespec,$(1),$(PKG_INSTALL_DIR),$$(1))
  154. $$(call Python3/DeleteNonSourceFiles,$$(1))
  155. $$(call Python3/DeleteEmptyDirs,$$(1))
  156. endef
  157. endif # Package/$(1)/install
  158. endef
  159. # Py3Build
  160. PYTHON3_PKG_SETUP_DIR ?=
  161. PYTHON3_PKG_SETUP_GLOBAL_ARGS ?=
  162. PYTHON3_PKG_SETUP_ARGS ?= --single-version-externally-managed
  163. PYTHON3_PKG_SETUP_VARS ?=
  164. PYTHON3_PKG_HOST_PIP_INSTALL_ARGS = \
  165. $(foreach req,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS), \
  166. --requirement \
  167. $(if $(findstring /,$(req)),$(req),$(python3_mk_path)host-pip-requirements/$(req).txt) \
  168. )
  169. define Py3Build/FindStdlibDepends
  170. $(SHELL) $(python3_mk_path)python3-find-stdlib-depends.sh -n "$(PKG_NAME)" "$(PKG_BUILD_DIR)"
  171. endef
  172. ifneq ($(strip $(PYPI_NAME)),)
  173. define Py3Build/CheckHostPipVersionMatch
  174. if grep -q "$(PYPI_NAME)==" $(python3_mk_path)host-pip-requirements/*.txt ; then \
  175. if ! grep -q "$(PYPI_NAME)==$(PKG_VERSION)" $(python3_mk_path)host-pip-requirements/*.txt ; then \
  176. printf "\nPlease update version of $(PYPI_NAME) to $(PKG_VERSION) in 'host-pip-requirements'/\n\n" ; \
  177. exit 1 ; \
  178. fi \
  179. fi
  180. endef
  181. endif
  182. define Py3Build/Compile/Default
  183. $(if $(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS), \
  184. $(call HostPython3/PipInstall,$(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS)) \
  185. )
  186. $(call Python3/ModSetup, \
  187. $(PYTHON3_PKG_SETUP_DIR), \
  188. $(PYTHON3_PKG_SETUP_GLOBAL_ARGS) \
  189. install --prefix="/usr" --root="$(PKG_INSTALL_DIR)" \
  190. $(PYTHON3_PKG_SETUP_ARGS), \
  191. $(PYTHON3_PKG_SETUP_VARS) \
  192. )
  193. endef
  194. Py3Build/Configure=$(Py3Build/Configure/Default)
  195. Py3Build/Compile=$(Py3Build/Compile/Default)
  196. PYTHON3_PKG_BUILD ?= 1
  197. ifeq ($(strip $(PYTHON3_PKG_BUILD)),1)
  198. ifeq ($(PY3),stdlib)
  199. Hooks/Configure/Post+=Py3Build/FindStdlibDepends
  200. endif
  201. Hooks/Configure/Post+=Py3Build/CheckHostPipVersionMatch
  202. Build/Compile=$(Py3Build/Compile)
  203. endif