From 20685f522067271662c8c425a12e4822de08093d Mon Sep 17 00:00:00 2001
From: Alexandru Ardelean <ardeleanalex@gmail.com>
Date: Thu, 2 Mar 2017 17:47:29 +0200
Subject: [PATCH] python: split source packages away from compiled packages

Well, they're not yet compiled, but in the next commit
they should be.

People have been complaining [citation needed] to me
via email or via Github that Python's performance is crap
because it packages sources directly and they're not compiled.
And Python has to compile the sources on each run, and
on-the-fly.

Allowing compilation caching is also a no-no, because
I'll get complaints that the flash storage fills up
whenever a Python app runs.

So, to give the user a choice, the new de-facto packaging
for Python packages will be:
* ship compiled + [ preferably ] optimized files
* package sources separately

The problem is that this doubles the number of packages
in LEDE/OpenWrt, but build-times should not suffer a big
hit, since the compilation is done once, and the
install phase should not be too intensive.

Oh, and people don't need ship source packages if
they don't want to.
To do that, a packager needs to just call
`$(eval $(call BuildPackage,python-<package>-src))`

The `python-` prefix is important.

Signed-off-by: Alexandru Ardelean <ardeleanalex@gmail.com>
---
 lang/python/Makefile                        |  5 +++
 lang/python/files/python-package-install.sh | 34 ++++++++++++++++++++-
 lang/python/files/python-package.mk         | 17 +++++++++++
 3 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/lang/python/Makefile b/lang/python/Makefile
index 25d07fbd4..a42a18700 100644
--- a/lang/python/Makefile
+++ b/lang/python/Makefile
@@ -301,6 +301,7 @@ $(eval $(call HostBuild))
 $(foreach package, $(PYTHON_PACKAGES),  \
 	$(eval $(call PyPackage,$(package))) \
 	$(eval $(call BuildPackage,$(package))) \
+	$(eval $(call BuildPackage,$(package)-src)) \
 )
 
 $(eval $(call PyPackage,python-base))
@@ -312,3 +313,7 @@ $(eval $(call BuildPackage,python-pip-conf))
 $(eval $(call BuildPackage,python-base))
 $(eval $(call BuildPackage,python-light))
 $(eval $(call BuildPackage,python))
+
+$(eval $(call BuildPackage,python-base-src))
+$(eval $(call BuildPackage,python-light-src))
+$(eval $(call BuildPackage,python-src))
diff --git a/lang/python/files/python-package-install.sh b/lang/python/files/python-package-install.sh
index c3de0cc95..a08f8b415 100644
--- a/lang/python/files/python-package-install.sh
+++ b/lang/python/files/python-package-install.sh
@@ -34,5 +34,37 @@ process_filespec() {
 	)
 }
 
-process_filespec "$1" "$2" "$3"
+src_dir="$1"
+dst_dir="$2"
+python="$3"
+mode="$4"
+filespec="$5"
+
+process_filespec "$src_dir" "$dst_dir" "$filespec" || {
+	echo "process filespec error-ed"
+	exit 1
+}
+
+if [ "$mode" == "sources" ] ; then
+	# Copy only python source files
+	find $dst_dir -not -name "*\.py" | xargs rm -f
+	# Delete empty folders (if the case)
+	find $dst_dir/usr -type d | xargs rmdir &> /dev/null
+	rmdir $dst_dir/usr &> /dev/null
+	exit 0
+fi
+
+# XXX [So that you won't goof as I did]
+# Note: Yes, I tried to use the -O & -OO flags here.
+#       However the generated byte-codes were not portable.
+#       So, we just stuck to un-optimized byte-codes,
+#       which is still way better/faster than running
+#       Python sources all the time.
+$python -m compileall -d '/' $dst_dir || {
+	echo "python -m compileall err-ed"
+	exit 1
+}
+# Delete source files and pyc [ un-optimized bytecode files ]
+# We may want to make this optimization thing configurable later, but not sure atm
+find $dst_dir -name "*\.py" | xargs rm -f
 
diff --git a/lang/python/files/python-package.mk b/lang/python/files/python-package.mk
index 4f05dfebb..e9ec85c6f 100644
--- a/lang/python/files/python-package.mk
+++ b/lang/python/files/python-package.mk
@@ -34,6 +34,17 @@ endif
 
 define PyPackage
 
+  define Package/$(1)-src
+    $(call Package/$(1))
+    TITLE+= (sources)
+    DEPENDS:=$$$$(foreach dep,$$$$(filter +python-%,$$$$(DEPENDS)),$$$$(dep)-src)
+  endef
+
+  define Package/$(1)-src/description
+    $(call Package/$(1)/description).
+    (Contains the Python sources for this package).
+  endef
+
   # Add default PyPackage filespec none defined
   ifndef PyPackage/$(1)/filespec
     define PyPackage/$(1)/filespec
@@ -58,16 +69,22 @@ define PyPackage
 	if [ -e files/python-package-install.sh ] ; then \
 		$(SHELL) files/python-package-install.sh \
 			"$(PKG_INSTALL_DIR)" "$$(1)" \
+			"$(HOST_PYTHON_BIN)" "$$(2)" \
 			"$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
 	elif [ -e $(STAGING_DIR)/mk/python-package-install.sh ] ; then \
 		$(SHELL) $(STAGING_DIR)/mk/python-package-install.sh \
 			"$(PKG_INSTALL_DIR)" "$$(1)" \
+			"$(HOST_PYTHON_BIN)" "$$(2)" \
 			"$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
 	else \
 		echo "No 'python-package-install.sh' script found" ; \
 		exit 1 ; \
 	fi
   endef
+
+  define Package/$(1)-src/install
+	$$(call Package/$(1)/install,$$(1),sources)
+  endef
 endef
 
 $(call include_mk, python-host.mk)