Merge pull request #11175 from jefferyto/python-fix-float-byte-order

python: Fix float byte order detection
4 years ago · 3a0034bfb6
--- a/lang/python/python/Makefile
+++ b/lang/python/python/Makefile
@ -12,7 +12,7 @@ include ../python-version.mk

 PKG_NAME:=python
 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
 PKG_RELEASE:=1
 PKG_RELEASE:=2

 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
 PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION)
--- a/lang/python/python/patches/026-bpo-34585-Dont-do-runtime-test-to-get-float-byte-order-GH-9085.patch
+++ b/lang/python/python/patches/026-bpo-34585-Dont-do-runtime-test-to-get-float-byte-order-GH-9085.patch
@ -0,0 +1,209 @@
 From 2a9c3805ddedf282881ef7811a561c70b74f80b1 Mon Sep 17 00:00:00 2001
 From: Ross Burton <ross@burtonini.com>
 Date: Wed, 19 Sep 2018 07:25:48 +0100
 Subject: [PATCH] closes bpo-34585: Don't do runtime test to get float byte
 order. (GH-9085)

 Currently configure.ac uses AC_RUN_IFELSE to determine the byte order of doubles, but this silently fails under cross compilation and Python doesn't do floats properly.

 Instead, steal a macro from autoconf-archive which compiles code using magic doubles (which encode to ASCII) and grep for the representation in the binary.

 RFC because this doesn't yet handle the weird ancient ARMv4 OABI 'mixed-endian' encoding properly. This encoding is ancient and I don't believe the union of "Python 3.8 users" and "OABI users" has anything in. Should the support for this just be dropped too? Alternatively, someone will need to find an OABI toolchain to verify the encoding of the magic double.
 ---
 .../2018-09-18-16-28-31.bpo-34585.CGMu0h.rst  |  3 +
 configure.ac                                  | 76 +++--------------
 m4/ax_c_float_words_bigendian.m4              | 83 +++++++++++++++++++
 3 files changed, 99 insertions(+), 63 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Build/2018-09-18-16-28-31.bpo-34585.CGMu0h.rst
 create mode 100644 m4/ax_c_float_words_bigendian.m4

 diff --git a/Misc/NEWS.d/next/Build/2018-09-18-16-28-31.bpo-34585.CGMu0h.rst b/Misc/NEWS.d/next/Build/2018-09-18-16-28-31.bpo-34585.CGMu0h.rst
 new file mode 100644
 index 0000000000000..01318e6e46a32
 --- /dev/null
 +++ b/Misc/NEWS.d/next/Build/2018-09-18-16-28-31.bpo-34585.CGMu0h.rst
@@ -0,0 +1,3 @@
 +Check for floating-point byte order in configure.ac using compilation tests
 +instead of executing code, so that these checks work in cross-compiled
 +builds.
 diff --git a/configure.ac b/configure.ac
 index 03638f8ae9bc7..96331ec221be2 100644
 --- a/configure.ac
 +++ b/configure.ac
@@ -4206,74 +4206,24 @@ fi
 # * Check for various properties of floating point *
 # **************************************************
 
 -AC_MSG_CHECKING(whether C doubles are little-endian IEEE 754 binary64)
 -AC_CACHE_VAL(ac_cv_little_endian_double, [
 -AC_RUN_IFELSE([AC_LANG_SOURCE([[
 -#include <string.h>
 -int main() {
 -    double x = 9006104071832581.0;
 -    if (memcmp(&x, "\x05\x04\x03\x02\x01\xff\x3f\x43", 8) == 0)
 -        return 0;
 -    else
 -        return 1;
 -}
 -]])],
 -[ac_cv_little_endian_double=yes],
 -[ac_cv_little_endian_double=no],
 -[ac_cv_little_endian_double=no])])
 -AC_MSG_RESULT($ac_cv_little_endian_double)
 -if test "$ac_cv_little_endian_double" = yes
 -then
 -  AC_DEFINE(DOUBLE_IS_LITTLE_ENDIAN_IEEE754, 1,
 -  [Define if C doubles are 64-bit IEEE 754 binary format, stored
 -   with the least significant byte first])
 -fi
 -
 -AC_MSG_CHECKING(whether C doubles are big-endian IEEE 754 binary64)
 -AC_CACHE_VAL(ac_cv_big_endian_double, [
 -AC_RUN_IFELSE([AC_LANG_SOURCE([[
 -#include <string.h>
 -int main() {
 -    double x = 9006104071832581.0;
 -    if (memcmp(&x, "\x43\x3f\xff\x01\x02\x03\x04\x05", 8) == 0)
 -        return 0;
 -    else
 -        return 1;
 -}
 -]])],
 -[ac_cv_big_endian_double=yes],
 -[ac_cv_big_endian_double=no],
 -[ac_cv_big_endian_double=no])])
 -AC_MSG_RESULT($ac_cv_big_endian_double)
 -if test "$ac_cv_big_endian_double" = yes
 +AX_C_FLOAT_WORDS_BIGENDIAN
 +if test "$ax_cv_c_float_words_bigendian" = "yes"
 then
   AC_DEFINE(DOUBLE_IS_BIG_ENDIAN_IEEE754, 1,
   [Define if C doubles are 64-bit IEEE 754 binary format, stored
    with the most significant byte first])
 -fi
 -
 -# Some ARM platforms use a mixed-endian representation for doubles.
 -# While Python doesn't currently have full support for these platforms
 -# (see e.g., issue 1762561), we can at least make sure that float <-> string
 -# conversions work.
 -AC_MSG_CHECKING(whether C doubles are ARM mixed-endian IEEE 754 binary64)
 -AC_CACHE_VAL(ac_cv_mixed_endian_double, [
 -AC_RUN_IFELSE([AC_LANG_SOURCE([[
 -#include <string.h>
 -int main() {
 -    double x = 9006104071832581.0;
 -    if (memcmp(&x, "\x01\xff\x3f\x43\x05\x04\x03\x02", 8) == 0)
 -        return 0;
 -    else
 -        return 1;
 -}
 -]])],
 -[ac_cv_mixed_endian_double=yes],
 -[ac_cv_mixed_endian_double=no],
 -[ac_cv_mixed_endian_double=no])])
 -AC_MSG_RESULT($ac_cv_mixed_endian_double)
 -if test "$ac_cv_mixed_endian_double" = yes
 +elif test "$ax_cv_c_float_words_bigendian" = "no"
 then
 +  AC_DEFINE(DOUBLE_IS_LITTLE_ENDIAN_IEEE754, 1,
 +  [Define if C doubles are 64-bit IEEE 754 binary format, stored
 +   with the least significant byte first])
 +else
 +  # Some ARM platforms use a mixed-endian representation for doubles.
 +  # While Python doesn't currently have full support for these platforms
 +  # (see e.g., issue 1762561), we can at least make sure that float <-> string
 +  # conversions work.
 +  # FLOAT_WORDS_BIGENDIAN doesnt actually detect this case, but if it's not big
 +  # or little, then it must be this?
   AC_DEFINE(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754, 1,
   [Define if C doubles are 64-bit IEEE 754 binary format, stored
    in ARM mixed-endian order (byte order 45670123)])
 diff --git a/m4/ax_c_float_words_bigendian.m4 b/m4/ax_c_float_words_bigendian.m4
 new file mode 100644
 index 0000000000000..216b90d803187
 --- /dev/null
 +++ b/m4/ax_c_float_words_bigendian.m4
@@ -0,0 +1,83 @@
 +# ===============================================================================
 +#  https://www.gnu.org/software/autoconf-archive/ax_c_float_words_bigendian.html
 +# ===============================================================================
 +#
 +# SYNOPSIS
 +#
 +#   AX_C_FLOAT_WORDS_BIGENDIAN([ACTION-IF-TRUE], [ACTION-IF-FALSE], [ACTION-IF-UNKNOWN])
 +#
 +# DESCRIPTION
 +#
 +#   Checks the ordering of words within a multi-word float. This check is
 +#   necessary because on some systems (e.g. certain ARM systems), the float
 +#   word ordering can be different from the byte ordering. In a multi-word
 +#   float context, "big-endian" implies that the word containing the sign
 +#   bit is found in the memory location with the lowest address. This
 +#   implementation was inspired by the AC_C_BIGENDIAN macro in autoconf.
 +#
 +#   The endianness is detected by first compiling C code that contains a
 +#   special double float value, then grepping the resulting object file for
 +#   certain strings of ASCII values. The double is specially crafted to have
 +#   a binary representation that corresponds with a simple string. In this
 +#   implementation, the string "noonsees" was selected because the
 +#   individual word values ("noon" and "sees") are palindromes, thus making
 +#   this test byte-order agnostic. If grep finds the string "noonsees" in
 +#   the object file, the target platform stores float words in big-endian
 +#   order. If grep finds "seesnoon", float words are in little-endian order.
 +#   If neither value is found, the user is instructed to specify the
 +#   ordering.
 +#
 +# LICENSE
 +#
 +#   Copyright (c) 2008 Daniel Amelang <dan@amelang.net>
 +#
 +#   Copying and distribution of this file, with or without modification, are
 +#   permitted in any medium without royalty provided the copyright notice
 +#   and this notice are preserved. This file is offered as-is, without any
 +#   warranty.
 +
 +#serial 11
 +
 +AC_DEFUN([AX_C_FLOAT_WORDS_BIGENDIAN],
 +  [AC_CACHE_CHECK(whether float word ordering is bigendian,
 +                  ax_cv_c_float_words_bigendian, [
 +
 +ax_cv_c_float_words_bigendian=unknown
 +AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
 +
 +double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0;
 +
 +]])], [
 +
 +if grep noonsees conftest.$ac_objext >/dev/null ; then
 +  ax_cv_c_float_words_bigendian=yes
 +fi
 +if grep seesnoon conftest.$ac_objext >/dev/null ; then
 +  if test "$ax_cv_c_float_words_bigendian" = unknown; then
 +    ax_cv_c_float_words_bigendian=no
 +  else
 +    ax_cv_c_float_words_bigendian=unknown
 +  fi
 +fi
 +
 +])])
 +
 +case $ax_cv_c_float_words_bigendian in
 +  yes)
 +    m4_default([$1],
 +      [AC_DEFINE([FLOAT_WORDS_BIGENDIAN], 1,
 +                 [Define to 1 if your system stores words within floats
 +                  with the most significant word first])]) ;;
 +  no)
 +    $2 ;;
 +  *)
 +    m4_default([$3],
 +      [AC_MSG_ERROR([
 +
 +Unknown float word ordering. You need to manually preset
 +ax_cv_c_float_words_bigendian=no (or yes) according to your system.
 +
 +    ])]) ;;
 +esac
 +
 +])# AX_C_FLOAT_WORDS_BIGENDIAN
--- a/lang/python/python/patches/027-bpo-34585-run-autoconf-GH-9411-edited.patch
+++ b/lang/python/python/patches/027-bpo-34585-run-autoconf-GH-9411-edited.patch
@ -0,0 +1,218 @@
 From b3b8cb419e496629873fa7dda82a01863f58617a Mon Sep 17 00:00:00 2001
 From: Benjamin Peterson <benjamin@python.org>
 Date: Tue, 18 Sep 2018 23:49:05 -0700
 Subject: [PATCH] run autoconf (GH-9411)

 Follow up to 2a9c3805ddedf282881ef7811a561c70b74f80b1 (bpo-34585).
 ---
 aclocal.m4    |   1 +
 configure     | 146 ++++++++++++++++----------------------------------
 pyconfig.h.in |   4 ++
 3 files changed, 51 insertions(+), 100 deletions(-)

 diff --git a/aclocal.m4 b/aclocal.m4
 index 6a24d8e6b9c00..030e6877de9f7 100644
 --- a/aclocal.m4
 +++ b/aclocal.m4
@@ -288,3 +288,4 @@ AS_VAR_COPY([$1], [pkg_cv_][$1])
 AS_VAR_IF([$1], [""], [$5], [$4])dnl
 ])dnl PKG_CHECK_VAR
 
 +m4_include([m4/ax_c_float_words_bigendian.m4])
 diff --git a/configure b/configure
 index 7b0c734b5e25e..38546d6ca7b40 100755
 --- a/configure
 +++ b/configure
@@ -13853,131 +13853,77 @@ fi
 # * Check for various properties of floating point *
 # **************************************************
 
 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C doubles are little-endian IEEE 754 binary64" >&5
 -$as_echo_n "checking whether C doubles are little-endian IEEE 754 binary64... " >&6; }
 -if ${ac_cv_little_endian_double+:} false; then :
 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether float word ordering is bigendian" >&5
 +$as_echo_n "checking whether float word ordering is bigendian... " >&6; }
 +if ${ax_cv_c_float_words_bigendian+:} false; then :
   $as_echo_n "(cached) " >&6
 else
 
 -if test "$cross_compiling" = yes; then :
 -  ac_cv_little_endian_double=no
 -else
 -  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 +
 +ax_cv_c_float_words_bigendian=unknown
 +cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 
 -#include <string.h>
 -int main() {
 -    double x = 9006104071832581.0;
 -    if (memcmp(&x, "\x05\x04\x03\x02\x01\xff\x3f\x43", 8) == 0)
 -        return 0;
 -    else
 -        return 1;
 -}
 
 -_ACEOF
 -if ac_fn_c_try_run "$LINENO"; then :
 -  ac_cv_little_endian_double=yes
 -else
 -  ac_cv_little_endian_double=no
 -fi
 -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
 -  conftest.$ac_objext conftest.beam conftest.$ac_ext
 -fi
 +double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0;
 
 -fi
 
 -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_little_endian_double" >&5
 -$as_echo "$ac_cv_little_endian_double" >&6; }
 -if test "$ac_cv_little_endian_double" = yes
 -then
 +_ACEOF
 +if ac_fn_c_try_compile "$LINENO"; then :
 
 -$as_echo "#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1" >>confdefs.h
 
 +if grep noonsees conftest.$ac_objext >/dev/null ; then
 +  ax_cv_c_float_words_bigendian=yes
 +fi
 +if grep seesnoon conftest.$ac_objext >/dev/null ; then
 +  if test "$ax_cv_c_float_words_bigendian" = unknown; then
 +    ax_cv_c_float_words_bigendian=no
 +  else
 +    ax_cv_c_float_words_bigendian=unknown
 +  fi
 fi
 
 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C doubles are big-endian IEEE 754 binary64" >&5
 -$as_echo_n "checking whether C doubles are big-endian IEEE 754 binary64... " >&6; }
 -if ${ac_cv_big_endian_double+:} false; then :
 -  $as_echo_n "(cached) " >&6
 -else
 -
 -if test "$cross_compiling" = yes; then :
 -  ac_cv_big_endian_double=no
 -else
 -  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 -/* end confdefs.h.  */
 -
 -#include <string.h>
 -int main() {
 -    double x = 9006104071832581.0;
 -    if (memcmp(&x, "\x43\x3f\xff\x01\x02\x03\x04\x05", 8) == 0)
 -        return 0;
 -    else
 -        return 1;
 -}
 
 -_ACEOF
 -if ac_fn_c_try_run "$LINENO"; then :
 -  ac_cv_big_endian_double=yes
 -else
 -  ac_cv_big_endian_double=no
 fi
 -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
 -  conftest.$ac_objext conftest.beam conftest.$ac_ext
 +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 fi
 +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_c_float_words_bigendian" >&5
 +$as_echo "$ax_cv_c_float_words_bigendian" >&6; }
 
 -fi
 +case $ax_cv_c_float_words_bigendian in
 +  yes)
 
 -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_big_endian_double" >&5
 -$as_echo "$ac_cv_big_endian_double" >&6; }
 -if test "$ac_cv_big_endian_double" = yes
 -then
 +$as_echo "#define FLOAT_WORDS_BIGENDIAN 1" >>confdefs.h
 + ;;
 +  no)
 +     ;;
 +  *)
 +    as_fn_error $? "
 
 -$as_echo "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h
 +Unknown float word ordering. You need to manually preset
 +ax_cv_c_float_words_bigendian=no (or yes) according to your system.
 
 -fi
 +    " "$LINENO" 5 ;;
 +esac
 
 -# Some ARM platforms use a mixed-endian representation for doubles.
 -# While Python doesn't currently have full support for these platforms
 -# (see e.g., issue 1762561), we can at least make sure that float <-> string
 -# conversions work.
 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C doubles are ARM mixed-endian IEEE 754 binary64" >&5
 -$as_echo_n "checking whether C doubles are ARM mixed-endian IEEE 754 binary64... " >&6; }
 -if ${ac_cv_mixed_endian_double+:} false; then :
 -  $as_echo_n "(cached) " >&6
 -else
 
 -if test "$cross_compiling" = yes; then :
 -  ac_cv_mixed_endian_double=no
 -else
 -  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 -/* end confdefs.h.  */
 +if test "$ax_cv_c_float_words_bigendian" = "yes"
 +then
 
 -#include <string.h>
 -int main() {
 -    double x = 9006104071832581.0;
 -    if (memcmp(&x, "\x01\xff\x3f\x43\x05\x04\x03\x02", 8) == 0)
 -        return 0;
 -    else
 -        return 1;
 -}
 +$as_echo "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h
 
 -_ACEOF
 -if ac_fn_c_try_run "$LINENO"; then :
 -  ac_cv_mixed_endian_double=yes
 -else
 -  ac_cv_mixed_endian_double=no
 -fi
 -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
 -  conftest.$ac_objext conftest.beam conftest.$ac_ext
 -fi
 +elif test "$ax_cv_c_float_words_bigendian" = "no"
 +then
 
 -fi
 +$as_echo "#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1" >>confdefs.h
 
 -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_mixed_endian_double" >&5
 -$as_echo "$ac_cv_mixed_endian_double" >&6; }
 -if test "$ac_cv_mixed_endian_double" = yes
 -then
 +else
 +  # Some ARM platforms use a mixed-endian representation for doubles.
 +  # While Python doesn't currently have full support for these platforms
 +  # (see e.g., issue 1762561), we can at least make sure that float <-> string
 +  # conversions work.
 +  # FLOAT_WORDS_BIGENDIAN doesnt actually detect this case, but if it's not big
 +  # or little, then it must be this?
 
 $as_echo "#define DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 1" >>confdefs.h
 
 diff --git a/pyconfig.h.in b/pyconfig.h.in
 index 360f79994fafe..41e0479cad2e3 100644
 --- a/pyconfig.h.in
 +++ b/pyconfig.h.in
@@ -30,6 +30,10 @@
 /* Define if --enable-ipv6 is specified */
 #undef ENABLE_IPV6
 
 +/* Define to 1 if your system stores words within floats with the most
 +   significant word first */
 +#undef FLOAT_WORDS_BIGENDIAN
 +
 /* Define if flock needs to be linked with bsd library. */
 #undef FLOCK_NEEDS_LIBBSD