Browse Source
python3: Add script to help find standard library dependencies
This adds a script that searches a Python package's source code to find
imports for separately-packaged standard library modules.
The script can be run by calling make with the configure target and
"PY3=stdlib V=s" arguments, e.g.
make package/python3-lxml/configure PY3=stdlib V=s
This also updates the readme on how to call this script, as well as more
information on Python package dependencies in general.
Signed-off-by: Jeffery To <jeffery.to@gmail.com>
lilik-openwrt-22.03
Jeffery To
5 years ago
3 changed files with 480 additions and 3 deletions
Unified View
Diff Options
-
+38 -3lang/python/README.md
-
+434 -0lang/python/python3-find-stdlib-depends.sh
-
+8 -0lang/python/python3-package.mk
+ 38
- 3
lang/python/README.md
View File
+ 434
- 0
lang/python/python3-find-stdlib-depends.sh
View File
| @ -0,0 +1,434 @@ | |||||
| #!/bin/sh | |||||
| # Packages data | |||||
| # | |||||
| # Notes: | |||||
| # * python3-codecs: Also contains codecs for CJK encodings but we don't | |||||
| # have a good way to check for uses | |||||
| # * python3-openssl: Don't include hashlib as it supports several | |||||
| # standard algorithms without requiring OpenSSL | |||||
| packages=" | |||||
| python3-asyncio: asyncio | |||||
| python3-cgi: cgi | |||||
| python3-cgitb: cgitb | |||||
| python3-codecs: unicodedata | |||||
| python3-ctypes: ctypes | |||||
| python3-dbm: dbm | |||||
| python3-decimal: decimal | |||||
| python3-distutils: distutils | |||||
| python3-email: email | |||||
| python3-gdbm: dbm.gnu | |||||
| python3-logging: logging | |||||
| python3-lzma: lzma | |||||
| python3-multiprocessing: multiprocessing | |||||
| python3-ncurses: ncurses | |||||
| python3-openssl: ssl | |||||
| python3-pydoc: doctest pydoc | |||||
| python3-sqlite3: sqlite3 | |||||
| python3-unittest: unittest | |||||
| python3-urllib: urllib | |||||
| python3-xml: xml xmlrpc | |||||
| " | |||||
| # Constants | |||||
| stdin_name="<stdin>" | |||||
| grep_dir_filters=" | |||||
| -Ir | |||||
| --include=*.py | |||||
| --exclude=setup.py | |||||
| --exclude=test_*.py | |||||
| --exclude=*_test.py | |||||
| --exclude-dir=test | |||||
| --exclude-dir=tests | |||||
| --exclude-dir=ipkg-* | |||||
| --exclude-dir=.pkgdir | |||||
| " | |||||
| log_level_notice=5 | |||||
| log_level_info=6 | |||||
| log_level_debug=7 | |||||
| # /usr/include/sysexits.h | |||||
| ex_usage=64 | |||||
| ex_noinput=66 | |||||
| ex_unavailable=69 | |||||
| ex_software=70 | |||||
| newline=" | |||||
| " | |||||
| oifs="$IFS" | |||||
| # Defaults | |||||
| grep_output_default_max_count=3 | |||||
| grep_output_default_color_when="auto" | |||||
| grep_output_default_line_prefix="-HnT --label=$stdin_name" | |||||
| grep_output_default_context_num=1 | |||||
| log_level_default="$log_level_info" | |||||
| # Global variables | |||||
| log_level= | |||||
| grep= | |||||
| grep_output_options= | |||||
| grep_output_description= | |||||
| stdin= | |||||
| output_name= | |||||
| is_first_search= | |||||
| # Logging | |||||
| log() { | |||||
| printf '%s\n' "$*" | |||||
| } | |||||
| can_log_notice() { | |||||
| [ "$log_level" -ge "$log_level_notice" ] | |||||
| } | |||||
| can_log_info() { | |||||
| [ "$log_level" -ge "$log_level_info" ] | |||||
| } | |||||
| can_log_debug() { | |||||
| [ "$log_level" -ge "$log_level_debug" ] | |||||
| } | |||||
| log_notice() { | |||||
| if can_log_notice; then | |||||
| log "$@" | |||||
| fi | |||||
| } | |||||
| log_info() { | |||||
| if can_log_info; then | |||||
| log "$@" | |||||
| fi | |||||
| } | |||||
| log_debug() { | |||||
| if can_log_debug; then | |||||
| log "$@" | |||||
| fi | |||||
| } | |||||
| log_error() { | |||||
| printf '%s\n' "Error: $*" >&2 | |||||
| } | |||||
| # Usage | |||||
| usage() { | |||||
| cat <<- EOF | |||||
| Usage: ${0##*/} [OPTION]... [FILE]... | |||||
| Search for imports of certain Python standard libraries in each FILE, | |||||
| generate a list of suggested package dependencies. | |||||
| With no FILE, or when FILE is -, read standard input. | |||||
| Options: | |||||
| -c WHEN use color in output; | |||||
| WHEN is 'always', 'never', or 'auto' (default: '$grep_output_default_color_when') | |||||
| -h display this help text and exit | |||||
| -m NUM show max NUM matches per package per file (default: $grep_output_default_max_count); | |||||
| use 0 to show all matches | |||||
| -n NAME when one or no FILE is given, use NAME instead of FILE in | |||||
| displayed information | |||||
| -q show suggested dependencies only | |||||
| -v show verbose output (also show all matches) | |||||
| -x NUM show NUM lines of context (default: $grep_output_default_context_num) | |||||
| EOF | |||||
| } | |||||
| # Imports search | |||||
| get_package_modules() { | |||||
| local line="$1" | |||||
| local field_num=0 | |||||
| local IFS=: | |||||
| for field in $line; do | |||||
| # strip leading and trailing whitespace | |||||
| field="${field#"${field%%[! ]*}"}" | |||||
| field="${field%"${field##*[! ]}"}" | |||||
| # set variables in search_path() | |||||
| if [ "$field_num" -eq 0 ]; then | |||||
| package="$field" | |||||
| field_num=1 | |||||
| elif [ "$field_num" -eq 1 ]; then | |||||
| modules="$field" | |||||
| field_num=2 | |||||
| else | |||||
| field_num=3 | |||||
| fi | |||||
| done | |||||
| if [ "$field_num" -ne 2 ] || [ -z "$package" ] || [ -z "$modules" ]; then | |||||
| log_error "invalid package data \"$line\"" | |||||
| exit "$ex_software" | |||||
| fi | |||||
| } | |||||
| search_path_for_modules() { | |||||
| local path="$1" | |||||
| local path_is_dir="$2" | |||||
| local path_is_stdin="$3" | |||||
| local package="$4" | |||||
| local modules="$5" | |||||
| local modules_regex | |||||
| local regex | |||||
| local remove_dir_prefix | |||||
| local grep_results | |||||
| local IFS="$oifs" | |||||
| log_debug " Looking for modules in $package ($modules)" | |||||
| modules_regex=$(printf '%s' "$modules" | sed -e 's/\./\\./g' -e 's/\s\+/|/g') | |||||
| regex="\b(import\s+($modules_regex)|from\s+($modules_regex)\S*\s+import)\b" | |||||
| if [ -n "$path_is_dir" ]; then | |||||
| remove_dir_prefix="s|^\(\(\x1b[[0-9;]*[mK]\)*\)$path|\1|" | |||||
| grep_results=$($grep $grep_output_options $grep_dir_filters -E "$regex" "$path") | |||||
| elif [ -n "$path_is_stdin" ]; then | |||||
| grep_results=$(printf '%s\n' "$stdin" | $grep $grep_output_options -E "$regex") | |||||
| else | |||||
| grep_results=$($grep $grep_output_options -E "$regex" "$path") | |||||
| fi | |||||
| if [ "$?" -ne 0 ]; then | |||||
| log_debug " No imports found" | |||||
| log_debug "" | |||||
| return 0 | |||||
| fi | |||||
| log_info " Found imports for: $modules" | |||||
| if can_log_info; then | |||||
| printf '%s\n' "$grep_results" | sed -e "$remove_dir_prefix" -e "s/^/ /" | |||||
| fi | |||||
| log_info "" | |||||
| # set variable in search_path() | |||||
| suggested="$suggested +$package" | |||||
| } | |||||
| search_path() { | |||||
| local path="$1" | |||||
| local name="$2" | |||||
| local path_is_dir | |||||
| local path_is_stdin | |||||
| local package | |||||
| local modules | |||||
| local suggested | |||||
| local IFS="$newline" | |||||
| if [ "$path" = "-" ]; then | |||||
| path_is_stdin=1 | |||||
| else | |||||
| if ! [ -e "$path" ]; then | |||||
| log_error "$path does not exist" | |||||
| exit "$ex_noinput" | |||||
| fi | |||||
| if [ -d "$path" ]; then | |||||
| path="${path%/}/" | |||||
| path_is_dir=1 | |||||
| fi | |||||
| fi | |||||
| log_info "" | |||||
| log_info "Searching $name (showing $grep_output_description):" | |||||
| log_info "" | |||||
| if [ -n "$path_is_stdin" ]; then | |||||
| stdin="$(cat)" | |||||
| fi | |||||
| for line in $packages; do | |||||
| # strip leading whitespace | |||||
| line="${line#"${line%%[! ]*}"}" | |||||
| # skip blank lines or comments (beginning with #) | |||||
| if [ -z "$line" ] || [ "$line" != "${line###}" ]; then | |||||
| continue | |||||
| fi | |||||
| package= | |||||
| modules= | |||||
| get_package_modules "$line" | |||||
| search_path_for_modules "$path" "$path_is_dir" "$path_is_stdin" "$package" "$modules" | |||||
| done | |||||
| log_notice "Suggested dependencies for $name:" | |||||
| if [ -z "$suggested" ]; then | |||||
| suggested="(none)" | |||||
| fi | |||||
| IFS="$oifs" | |||||
| for package in $suggested; do | |||||
| log_notice " $package" | |||||
| done | |||||
| log_notice "" | |||||
| } | |||||
| # Find GNU grep | |||||
| if ggrep --version 2>&1 | grep -q GNU; then | |||||
| grep="ggrep" | |||||
| elif grep --version 2>&1 | grep -q GNU; then | |||||
| grep="grep" | |||||
| else | |||||
| log_error "cannot find GNU grep" | |||||
| exit "$ex_unavailable" | |||||
| fi | |||||
| # Process environment variables | |||||
| case $PYTHON3_FIND_STDLIB_DEPENDS_LOG_LEVEL in | |||||
| notice) | |||||
| log_level="$log_level_notice" | |||||
| ;; | |||||
| info) | |||||
| log_level="$log_level_info" | |||||
| ;; | |||||
| debug) | |||||
| log_level="$log_level_debug" | |||||
| ;; | |||||
| *) | |||||
| log_level="$log_level_default" | |||||
| ;; | |||||
| esac | |||||
| grep_output_max_count="${PYTHON3_FIND_STDLIB_DEPENDS_MAX_COUNT:-$grep_output_default_max_count}" | |||||
| grep_output_color_when="${PYTHON3_FIND_STDLIB_DEPENDS_COLOR_WHEN:-$grep_output_default_color_when}" | |||||
| grep_output_line_prefix="${PYTHON3_FIND_STDLIB_DEPENDS_LINE_PREFIX:-$grep_output_default_line_prefix}" | |||||
| grep_output_context_num="${PYTHON3_FIND_STDLIB_DEPENDS_CONTEXT_NUM:-$grep_output_default_context_num}" | |||||
| # Process command line options | |||||
| while getopts c:hm:n:qvx: OPT; do | |||||
| case $OPT in | |||||
| c) | |||||
| grep_output_color_when="$OPTARG" | |||||
| ;; | |||||
| h) | |||||
| usage | |||||
| exit 0 | |||||
| ;; | |||||
| m) | |||||
| grep_output_max_count="$OPTARG" | |||||
| ;; | |||||
| n) | |||||
| output_name="$OPTARG" | |||||
| ;; | |||||
| q) | |||||
| log_level="$log_level_notice" | |||||
| ;; | |||||
| v) | |||||
| log_level="$log_level_debug" | |||||
| ;; | |||||
| x) | |||||
| grep_output_context_num="$OPTARG" | |||||
| ;; | |||||
| \?) | |||||
| usage | |||||
| exit "$ex_usage" | |||||
| ;; | |||||
| esac | |||||
| done | |||||
| shift $((OPTIND - 1)) | |||||
| # Set up grep output options | |||||
| if can_log_info; then | |||||
| if [ "$grep_output_color_when" = "auto" ]; then | |||||
| if [ -t 1 ]; then | |||||
| grep_output_color_when="always" | |||||
| else | |||||
| grep_output_color_when="never" | |||||
| fi | |||||
| fi | |||||
| if ! can_log_debug && [ "$grep_output_max_count" -gt 0 ]; then | |||||
| grep_output_options="-m $grep_output_max_count" | |||||
| if [ "$grep_output_max_count" -eq 1 ]; then | |||||
| grep_output_description="max 1 match per file" | |||||
| else | |||||
| grep_output_description="max $grep_output_max_count matches per file" | |||||
| fi | |||||
| else | |||||
| grep_output_description="all matches" | |||||
| fi | |||||
| if [ "$grep_output_context_num" -gt 0 ]; then | |||||
| grep_output_options="$grep_output_options -C $grep_output_context_num" | |||||
| if [ "$grep_output_context_num" -eq 1 ]; then | |||||
| grep_output_description="$grep_output_description, 1 line of context" | |||||
| else | |||||
| grep_output_description="$grep_output_description, $grep_output_context_num lines of context" | |||||
| fi | |||||
| fi | |||||
| grep_output_options="$grep_output_options --color=$grep_output_color_when $grep_output_line_prefix" | |||||
| else | |||||
| grep_output_options="-q" | |||||
| fi | |||||
| # Main | |||||
| if [ "$#" -gt 0 ]; then | |||||
| is_first_search=1 | |||||
| if [ "$#" -gt 1 ]; then | |||||
| output_name= | |||||
| fi | |||||
| for path; do | |||||
| if [ -z "$is_first_search" ]; then | |||||
| log_info "====" | |||||
| fi | |||||
| if [ -z "$output_name" ]; then | |||||
| if [ "$path" = "-" ]; then | |||||
| output_name="$stdin_name" | |||||
| else | |||||
| output_name="$path" | |||||
| fi | |||||
| fi | |||||
| search_path "$path" "$output_name" | |||||
| is_first_search= | |||||
| output_name= | |||||
| done | |||||
| else | |||||
| search_path "-" "${output_name:-$stdin_name}" | |||||
| fi | |||||