Merge pull request #4130 from commodo/python-package-src

python, python3: ship packages with byte-codes and derive python source packages

lang/python/Makefile

@@ -12,7 +12,7 @@ include ./files/python-version.mk
 
 PKG_NAME:=python
 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
-PKG_RELEASE:=4
+PKG_RELEASE:=5
 
 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
 PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION)
@@ -104,6 +104,8 @@ define PyBasePackage
   define PyPackage/$(1)/filespec
     ifneq ($(2),)
       $(subst $(space),$(newline),$(foreach lib_file,$(2),+|$(lib_file)))
+      -|/usr/lib/python$(PYTHON_VERSION)/*/test
+      -|/usr/lib/python$(PYTHON_VERSION)/*/tests
     endif
   endef
 endef
@@ -193,6 +195,7 @@ define Build/InstallDev
 		./files/python-package.mk \
 		./files/python-host.mk \
 		./files/python-version.mk \
+		./files/python-package-install.sh \
 		$(STAGING_DIR)/mk/
 	$(CP) \
 		$(PKG_INSTALL_DIR)/usr/include/python$(PYTHON_VERSION) \
@@ -258,6 +261,7 @@ $(subst $(space),$(newline),$(foreach lib_file,$(PYTHON_LIB_FILES_DEL),-|$(lib_f
 endef
 
 define PyPackage/python-base/install
+	$(INSTALL_DIR) $(1)/usr/bin $(1)/usr/lib
 	$(LN) python$(PYTHON_VERSION) $(1)/usr/bin/python
 	$(LN) python$(PYTHON_VERSION) $(1)/usr/bin/python2
 	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libpython$(PYTHON_VERSION).so* $(1)/usr/lib/
@@ -297,6 +301,7 @@ $(eval $(call HostBuild))
 $(foreach package, $(PYTHON_PACKAGES),  \
 	$(eval $(call PyPackage,$(package))) \
 	$(eval $(call BuildPackage,$(package))) \
+	$(eval $(call BuildPackage,$(package)-src)) \
 )
 
 $(eval $(call PyPackage,python-base))
@@ -308,3 +313,7 @@ $(eval $(call BuildPackage,python-pip-conf))
 $(eval $(call BuildPackage,python-base))
 $(eval $(call BuildPackage,python-light))
 $(eval $(call BuildPackage,python))
+
+$(eval $(call BuildPackage,python-base-src))
+$(eval $(call BuildPackage,python-light-src))
+$(eval $(call BuildPackage,python-src))

lang/python/files/python-package-dev.mk

@@ -12,9 +12,9 @@ $(call Package/python/Default)
 endef
 
 define PyPackage/python-dev/install
-	$(INSTALL_DIR) $(1)/usr/bin
+	$(INSTALL_DIR) $(1)/usr/bin $(1)/usr/lib
 	$(CP) $(PKG_INSTALL_DIR)/usr/bin/python*config $(1)/usr/bin
-	$(CP) $(PKG_INSTALL_DIR)/usr/lib/libpython$(PYTHON_VERSION).so* $(1)/usr/lib
+	$(CP) $(PKG_INSTALL_DIR)/usr/lib/python$(PYTHON_VERSION)/config/libpython$(PYTHON_VERSION).a $(1)/usr/lib
 endef
 
 $(eval $(call PyBasePackage,python-dev, \

lang/python/files/python-package-install.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+
+process_filespec() {
+	local src_dir="$1"
+	local dst_dir="$2"
+	local filespec="$3"
+	echo "$filespec" | (
+	IFS='|'
+	while read fop fspec fperm; do
+		local fop=`echo "$fop" | tr -d ' \t\n'`
+		if [ "$fop" = "+" ]; then
+			if [ ! -e "${src_dir}${fspec}" ]; then
+				echo "File not found '${src_dir}${fspec}'"
+				exit 1
+			fi
+			dpath=`dirname "$fspec"`
+			if [ -z "$fperm" ]; then
+				dperm=`stat -c "%a" ${src_dir}${dpath}`
+			fi
+			mkdir -p -m$dperm ${dst_dir}${dpath}
+			echo "copying: '$fspec'"
+			cp -fpR ${src_dir}${fspec} ${dst_dir}${dpath}/
+			if [ -n "$fperm" ]; then
+				chmod -R $fperm ${dst_dir}${fspec}
+			fi
+		elif [ "$fop" = "-" ]; then
+			echo "removing: '$fspec'"
+			rm -fR ${dst_dir}${fspec}
+		elif [ "$fop" = "=" ]; then
+			echo "setting permissions: '$fperm' on '$fspec'"
+			chmod -R $fperm ${dst_dir}${fspec}
+		fi
+	done
+	)
+}
+
+src_dir="$1"
+dst_dir="$2"
+python="$3"
+mode="$4"
+filespec="$5"
+
+process_filespec "$src_dir" "$dst_dir" "$filespec" || {
+	echo "process filespec error-ed"
+	exit 1
+}
+
+if [ "$mode" == "sources" ] ; then
+	# Copy only python source files
+	find $dst_dir -not -name "*\.py" | xargs rm -f
+	# Delete empty folders (if the case)
+	find $dst_dir/usr -type d | xargs rmdir &> /dev/null
+	rmdir $dst_dir/usr &> /dev/null
+	exit 0
+fi
+
+# XXX [So that you won't goof as I did]
+# Note: Yes, I tried to use the -O & -OO flags here.
+#       However the generated byte-codes were not portable.
+#       So, we just stuck to un-optimized byte-codes,
+#       which is still way better/faster than running
+#       Python sources all the time.
+$python -m compileall -d '/' $dst_dir || {
+	echo "python -m compileall err-ed"
+	exit 1
+}
+# Delete source files and pyc [ un-optimized bytecode files ]
+# We may want to make this optimization thing configurable later, but not sure atm
+find $dst_dir -name "*\.py" | xargs rm -f
+

lang/python/files/python-package.mk

@@ -34,6 +34,17 @@ endif
 
 define PyPackage
 
+  define Package/$(1)-src
+    $(call Package/$(1))
+    TITLE+= (sources)
+    DEPENDS:=$$$$(foreach dep,$$$$(filter +python-%,$$$$(DEPENDS)),$$$$(dep)-src)
+  endef
+
+  define Package/$(1)-src/description
+    $(call Package/$(1)/description).
+    (Contains the Python sources for this package).
+  endef
+
   # Add default PyPackage filespec none defined
   ifndef PyPackage/$(1)/filespec
     define PyPackage/$(1)/filespec
@@ -44,8 +55,8 @@ define PyPackage
   ifndef PyPackage/$(1)/install
     define PyPackage/$(1)/install
 		if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \
-			$(INSTALL_DIR) $$(1)/usr/bin \
-			$(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/
+			$(INSTALL_DIR) $$(1)/usr/bin ; \
+			$(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \
 		fi
     endef
   endif
@@ -53,38 +64,26 @@ define PyPackage
   $(call shexport,PyPackage/$(1)/filespec)
 
   define Package/$(1)/install
-	find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
-	@echo "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" | ( \
-		IFS='|'; \
-		while read fop fspec fperm; do \
-		  fop=`echo "$$$$$$$$fop" | tr -d ' \t\n'`; \
-		  if [ "$$$$$$$$fop" = "+" ]; then \
-			if [ ! -e "$(PKG_INSTALL_DIR)$$$$$$$$fspec" ]; then \
-			  echo "File not found '$(PKG_INSTALL_DIR)$$$$$$$$fspec'"; \
-			  exit 1; \
-			fi; \
-			dpath=`dirname "$$$$$$$$fspec"`; \
-			if [ -n "$$$$$$$$fperm" ]; then \
-			  dperm="-m$$$$$$$$fperm"; \
-			else \
-			  dperm=`stat -c "%a" $(PKG_INSTALL_DIR)$$$$$$$$dpath`; \
-			fi; \
-			mkdir -p $$$$$$$$$dperm $$(1)$$$$$$$$dpath; \
-			echo "copying: '$$$$$$$$fspec'"; \
-			cp -fpR $(PKG_INSTALL_DIR)$$$$$$$$fspec $$(1)$$$$$$$$dpath/; \
-			if [ -n "$$$$$$$$fperm" ]; then \
-			  chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
-			fi; \
-		  elif [ "$$$$$$$$fop" = "-" ]; then \
-			echo "removing: '$$$$$$$$fspec'"; \
-			rm -fR $$(1)$$$$$$$$fspec; \
-		  elif [ "$$$$$$$$fop" = "=" ]; then \
-			echo "setting permissions: '$$$$$$$$fperm' on '$$$$$$$$fspec'"; \
-			chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
-		  fi; \
-		done; \
-	)
 	$(call PyPackage/$(1)/install,$$(1))
+	find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
+	if [ -e files/python-package-install.sh ] ; then \
+		$(SHELL) files/python-package-install.sh \
+			"$(PKG_INSTALL_DIR)" "$$(1)" \
+			"$(HOST_PYTHON_BIN)" "$$(2)" \
+			"$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
+	elif [ -e $(STAGING_DIR)/mk/python-package-install.sh ] ; then \
+		$(SHELL) $(STAGING_DIR)/mk/python-package-install.sh \
+			"$(PKG_INSTALL_DIR)" "$$(1)" \
+			"$(HOST_PYTHON_BIN)" "$$(2)" \
+			"$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
+	else \
+		echo "No 'python-package-install.sh' script found" ; \
+		exit 1 ; \
+	fi
+  endef
+
+  define Package/$(1)-src/install
+	$$(call Package/$(1)/install,$$(1),sources)
   endef
 endef
 
@@ -121,7 +120,7 @@ define Build/Compile/PyMod
 		cd $(PKG_BUILD_DIR)/$(strip $(1)), \
 		./setup.py $(2), \
 		$(3))
-	find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
+	find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
 endef
 
 define PyBuild/Compile/Default

lang/python3/Makefile

@@ -14,7 +14,7 @@ PYTHON_VERSION:=$(PYTHON3_VERSION)
 PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO)
 
 PKG_NAME:=python3
-PKG_RELEASE:=4
+PKG_RELEASE:=5
 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
 
 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
@@ -98,6 +98,8 @@ define Py3BasePackage
   define Py3Package/$(1)/filespec
     ifneq ($(2),)
       $(subst $(space),$(newline),$(foreach lib_file,$(2),+|$(lib_file)))
+      -|/usr/lib/python$(PYTHON_VERSION)/*/test
+      -|/usr/lib/python$(PYTHON_VERSION)/*/tests
     endif
   endef
 endef
@@ -187,6 +189,7 @@ define Build/InstallDev
 		./files/python3-package.mk \
 		./files/python3-host.mk \
 		./files/python3-version.mk \
+		./files/python3-package-install.sh \
 		$(STAGING_DIR)/mk/
 	$(CP) \
 		$(PKG_INSTALL_DIR)/usr/include/python$(PYTHON_VERSION) \
@@ -286,6 +289,7 @@ $(eval $(call HostBuild))
 $(foreach package, $(PYTHON3_PACKAGES),  \
 	$(eval $(call Py3Package,$(package))) \
 	$(eval $(call BuildPackage,$(package))) \
+	$(eval $(call BuildPackage,$(package)-src)) \
 )
 
 $(eval $(call Py3Package,python3-base))
@@ -295,3 +299,7 @@ $(eval $(call Py3Package,python3))
 $(eval $(call BuildPackage,python3-base))
 $(eval $(call BuildPackage,python3-light))
 $(eval $(call BuildPackage,python3))
+
+$(eval $(call BuildPackage,python3-base-src))
+$(eval $(call BuildPackage,python3-light-src))
+$(eval $(call BuildPackage,python3-src))

lang/python3/files/python3-package-dev.mk

@@ -12,10 +12,10 @@ $(call Package/python3/Default)
 endef
 
 define Py3Package/python3-dev/install
-	$(INSTALL_DIR) $(1)/usr/bin
+	$(INSTALL_DIR) $(1)/usr/bin $(1)/usr/lib
 	$(CP) $(PKG_INSTALL_DIR)/usr/bin/python$(PYTHON3_VERSION)-config $(1)/usr/bin
 	$(LN) python$(PYTHON3_VERSION)-config $(1)/usr/bin/python3-config
-	$(LN) python$(PYTHON_VERSION)/config-$(PYTHON_VERSION)/libpython$(PYTHON3_VERSION).a $(1)/usr/lib/libpython$(PYTHON3_VERSION).a
+	$(LN) python$(PYTHON_VERSION)/config-$(PYTHON_VERSION)/libpython$(PYTHON3_VERSION).a $(1)/usr/lib/
 endef
 
 $(eval $(call Py3BasePackage,python3-dev, \

lang/python3/files/python3-package-install.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+process_filespec() {
+	local src_dir="$1"
+	local dst_dir="$2"
+	local filespec="$3"
+	echo "$filespec" | (
+	IFS='|'
+	while read fop fspec fperm; do
+		local fop=`echo "$fop" | tr -d ' \t\n'`
+		if [ "$fop" = "+" ]; then
+			if [ ! -e "${src_dir}${fspec}" ]; then
+				echo "File not found '${src_dir}${fspec}'"
+				exit 1
+			fi
+			dpath=`dirname "$fspec"`
+			if [ -z "$fperm" ]; then
+				dperm=`stat -c "%a" ${src_dir}${dpath}`
+			fi
+			mkdir -p -m$dperm ${dst_dir}${dpath}
+			echo "copying: '$fspec'"
+			cp -fpR ${src_dir}${fspec} ${dst_dir}${dpath}/
+			if [ -n "$fperm" ]; then
+				chmod -R $fperm ${dst_dir}${fspec}
+			fi
+		elif [ "$fop" = "-" ]; then
+			echo "removing: '$fspec'"
+			rm -fR ${dst_dir}${fspec}
+		elif [ "$fop" = "=" ]; then
+			echo "setting permissions: '$fperm' on '$fspec'"
+			chmod -R $fperm ${dst_dir}${fspec}
+		fi
+	done
+	)
+}
+
+src_dir="$1"
+dst_dir="$2"
+python="$3"
+mode="$4"
+filespec="$5"
+
+process_filespec "$src_dir" "$dst_dir" "$filespec" || {
+	echo "process filespec error-ed"
+	exit 1
+}
+
+if [ "$mode" == "sources" ] ; then
+	# Copy only python source files
+	find $dst_dir -not -name "*\.py" | xargs rm -f
+	# Delete empty folders (if the case)
+	find $dst_dir/usr -type d | xargs rmdir &> /dev/null
+	rmdir $dst_dir/usr &> /dev/null
+	exit 0
+fi
+
+# XXX [So that you won't goof as I did]
+# Note: Yes, I tried to use the -O & -OO flags here.
+#       However the generated byte-codes were not portable.
+#       So, we just stuck to un-optimized byte-codes,
+#       which is still way better/faster than running
+#       Python sources all the time.
+$python -m compileall -b -d '/' $dst_dir || {
+	echo "python -m compileall err-ed"
+	exit 1
+}
+# Delete source files and pyc [ un-optimized bytecode files ]
+# We may want to make this optimization thing configurable later, but not sure atm
+find $dst_dir -name "*\.py" | xargs rm -f

lang/python3/files/python3-package.mk

@@ -34,6 +34,17 @@ endif
 
 define Py3Package
 
+  define Package/$(1)-src
+    $(call Package/$(1))
+    TITLE+= (sources)
+    DEPENDS:=$$$$(foreach dep,$$$$(filter +python3-%,$$$$(DEPENDS)),$$$$(dep)-src)
+  endef
+
+  define Package/$(1)-src/description
+    $(call Package/$(1)/description).
+    (Contains the Python3 sources for this package).
+  endef
+
   # Add default PyPackage filespec none defined
   ifndef Py3Package/$(1)/filespec
     define Py3Package/$(1)/filespec
@@ -44,8 +55,8 @@ define Py3Package
   ifndef Py3Package/$(1)/install
     define Py3Package/$(1)/install
 		if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \
-			$(INSTALL_DIR) $$(1)/usr/bin \
-			$(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/
+			$(INSTALL_DIR) $$(1)/usr/bin ; \
+			$(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \
 		fi
     endef
   endif
@@ -53,38 +64,26 @@ define Py3Package
   $(call shexport,Py3Package/$(1)/filespec)
 
   define Package/$(1)/install
-	find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
-	@echo "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" | ( \
-		IFS='|'; \
-		while read fop fspec fperm; do \
-		  fop=`echo "$$$$$$$$fop" | tr -d ' \t\n'`; \
-		  if [ "$$$$$$$$fop" = "+" ]; then \
-			if [ ! -e "$(PKG_INSTALL_DIR)$$$$$$$$fspec" ]; then \
-			  echo "File not found '$(PKG_INSTALL_DIR)$$$$$$$$fspec'"; \
-			  exit 1; \
-			fi; \
-			dpath=`dirname "$$$$$$$$fspec"`; \
-			if [ -n "$$$$$$$$fperm" ]; then \
-			  dperm="-m$$$$$$$$fperm"; \
-			else \
-			  dperm=`stat -c "%a" $(PKG_INSTALL_DIR)$$$$$$$$dpath`; \
-			fi; \
-			mkdir -p $$$$$$$$$dperm $$(1)$$$$$$$$dpath; \
-			echo "copying: '$$$$$$$$fspec'"; \
-			cp -fpR $(PKG_INSTALL_DIR)$$$$$$$$fspec $$(1)$$$$$$$$dpath/; \
-			if [ -n "$$$$$$$$fperm" ]; then \
-			  chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
-			fi; \
-		  elif [ "$$$$$$$$fop" = "-" ]; then \
-			echo "removing: '$$$$$$$$fspec'"; \
-			rm -fR $$(1)$$$$$$$$fspec; \
-		  elif [ "$$$$$$$$fop" = "=" ]; then \
-			echo "setting permissions: '$$$$$$$$fperm' on '$$$$$$$$fspec'"; \
-			chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
-		  fi; \
-		done; \
-	)
 	$(call Py3Package/$(1)/install,$$(1))
+	find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
+	if [ -e files/python3-package-install.sh ] ; then \
+		$(SHELL) files/python3-package-install.sh \
+			"$(PKG_INSTALL_DIR)" "$$(1)" \
+			"$(HOST_PYTHON3_BIN)" "$$(2)" \
+			"$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" ; \
+	elif [ -e $(STAGING_DIR)/mk/python3-package-install.sh ] ; then \
+		$(SHELL) $(STAGING_DIR)/mk/python3-package-install.sh \
+			"$(PKG_INSTALL_DIR)" "$$(1)" \
+			"$(HOST_PYTHON3_BIN)" "$$(2)" \
+			"$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" ; \
+	else \
+		echo "No 'python3-package-install.sh' script found" ; \
+		exit 1 ; \
+	fi
+  endef
+
+  define Package/$(1)-src/install
+	$$(call Package/$(1)/install,$$(1),sources)
   endef
 endef
 
@@ -121,7 +120,7 @@ define Build/Compile/Py3Mod
 		cd $(PKG_BUILD_DIR)/$(strip $(1)), \
 		./setup.py $(2), \
 		$(3))
-	find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
+	find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
 endef
 
 define Py3Build/Compile/Default