From 85c805d0ada334c0b90c3b74bc4b7c5779627e11 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Thu, 4 Apr 2019 02:45:19 +0800 Subject: [PATCH] python,python3: Increase max recursion level when generating bytecode "python -m compileall" has a default maximum recursion level of 10, i.e. it will descend up to 10 levels of subdirectories when looking for source files to compile. This is usually sufficient but there are packages that include more than 10 levels (botocore, https://github.com/openwrt/packages/pull/8214#discussion_r270056741). This adds the "-r" command line option to the call to compileall to increase the max recursion level (currently set to 20). This also patches Python 2's compileall.py to add this max recursion level option. (Python 3's compileall.py already supports this option.) This also applies some related changes to python-package-install.sh: * Use the "-delete" option with find instead of exec'ing rm / rmdir. For the case of removing empty directories (in delete_empty_dirs()), this has the added benefit of simplifying the code, as the "-delete" option implies "-depth", and thus find "does the right thing" (removing empty directories depth-first). * Remove the backslash in "-name" patterns (for find), as they are not regular expression but glob patterns. Signed-off-by: Jeffery To --- lang/python/python-package-install.sh | 16 ++++----- lang/python/python/Makefile | 2 +- .../021-compileall-add-recursion-option.patch | 33 +++++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 lang/python/python/patches/021-compileall-add-recursion-option.patch diff --git a/lang/python/python-package-install.sh b/lang/python/python-package-install.sh index 6b4fe54b1..2ca2abc16 100644 --- a/lang/python/python-package-install.sh +++ b/lang/python/python-package-install.sh @@ -43,11 +43,7 @@ process_filespec() { delete_empty_dirs() { local dst_dir="$1" if [ -d "$dst_dir/usr" ] ; then - for _ in $(seq 1 10) ; do - find "$dst_dir/usr" -empty -type d -exec rmdir {} \; || continue - break - done - rmdir "$dst_dir/usr" || true + find "$dst_dir/usr" -empty -type d -delete fi } @@ -60,7 +56,7 @@ filespec="$6" SED="${SED:-sed -e}" -find "$src_dir" -name "*\.exe" -exec rm -f {} \; +find "$src_dir" -name "*.exe" -delete process_filespec "$src_dir" "$dst_dir" "$filespec" || { echo "process filespec error-ed" @@ -75,7 +71,7 @@ fi if [ "$mode" == "sources" ] ; then # Copy only python source files - find "$dst_dir" -not -type d -not -name "*\.py" -exec rm -f {} \; + find "$dst_dir" -not -type d -not -name "*.py" -delete delete_empty_dirs "$dst_dir" exit 0 @@ -83,6 +79,8 @@ fi legacy= [ "$ver" == "3" ] && legacy="-b" +# default max recursion is 10 +max_recursion_level=20 # XXX [So that you won't goof as I did] # Note: Yes, I tried to use the -O & -OO flags here. @@ -90,14 +88,14 @@ legacy= # So, we just stuck to un-optimized byte-codes, # which is still way better/faster than running # Python sources all the time. -$python -m compileall $legacy -d '/' "$dst_dir" || { +$python -m compileall -r "$max_recursion_level" $legacy -d '/' "$dst_dir" || { echo "python -m compileall err-ed" exit 1 } # Delete source files and pyc [ un-optimized bytecode files ] # We may want to make this optimization thing configurable later, but not sure atm -find "$dst_dir" -type f -name "*\.py" -exec rm -f {} \; +find "$dst_dir" -type f -name "*.py" -delete delete_empty_dirs "$dst_dir" diff --git a/lang/python/python/Makefile b/lang/python/python/Makefile index 453220b9b..af584bcdd 100644 --- a/lang/python/python/Makefile +++ b/lang/python/python/Makefile @@ -12,7 +12,7 @@ include ../python-version.mk PKG_NAME:=python PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO) -PKG_RELEASE:=3 +PKG_RELEASE:=4 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION) diff --git a/lang/python/python/patches/021-compileall-add-recursion-option.patch b/lang/python/python/patches/021-compileall-add-recursion-option.patch new file mode 100644 index 000000000..fa75a4285 --- /dev/null +++ b/lang/python/python/patches/021-compileall-add-recursion-option.patch @@ -0,0 +1,33 @@ +diff --git a/Lib/compileall.py b/Lib/compileall.py +index 5cfa8bed3f..8716c9c0ca 100644 +--- a/Lib/compileall.py ++++ b/Lib/compileall.py +@@ -152,10 +152,10 @@ def main(): + """Script main program.""" + import getopt + try: +- opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:') ++ opts, args = getopt.getopt(sys.argv[1:], 'lr:fqd:x:i:') + except getopt.error, msg: + print msg +- print "usage: python compileall.py [-l] [-f] [-q] [-d destdir] " \ ++ print "usage: python compileall.py [-l] [-r recursion] [-f] [-q] [-d destdir] " \ + "[-x regexp] [-i list] [directory|file ...]" + print + print "arguments: zero or more file and directory names to compile; " \ +@@ -164,6 +164,7 @@ def main(): + print + print "options:" + print "-l: don't recurse into subdirectories" ++ print "-r recursion: control the maximum recursion level" + print "-f: force rebuild even if timestamps are up-to-date" + print "-q: output only error messages" + print "-d destdir: directory to prepend to file paths for use in " \ +@@ -187,6 +188,7 @@ def main(): + flist = None + for o, a in opts: + if o == '-l': maxlevels = 0 ++ if o == '-r': maxlevels = int(a) + if o == '-d': ddir = a + if o == '-f': force = 1 + if o == '-q': quiet = 1