Merge pull request #9131 from jefferyto/python-bpo-35907

python,python3: Fix CVE-2019-9948 - local_file:// allowed in urllib

lang/python/python/Makefile

@@ -12,7 +12,7 @@ include ../python-version.mk
 
 PKG_NAME:=python
 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
-PKG_RELEASE:=6
+PKG_RELEASE:=7
 
 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
 PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION)

lang/python/python/patches/023-bpo-35907-Avoid-file-reading-as-disallowing-the-unnecessary-URL-scheme-in-urllib-GH-11842.patch

@@ -0,0 +1,192 @@
+From 8f99cc799e4393bf1112b9395b2342f81b3f45ef Mon Sep 17 00:00:00 2001
+From: push0ebp <push0ebp@shl-MacBook-Pro.local>
+Date: Thu, 14 Feb 2019 02:05:46 +0900
+Subject: [PATCH 1/6] bpo-35907: Avoid file reading as disallowing the
+ unnecessary URL scheme in urllib
+
+---
+ Lib/test/test_urllib.py | 12 ++++++++++++
+ Lib/urllib.py           |  5 ++++-
+ 2 files changed, 16 insertions(+), 1 deletion(-)
+
+diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
+index 1ce9201c0693..e5f210e62a18 100644
+--- a/Lib/test/test_urllib.py
++++ b/Lib/test/test_urllib.py
+@@ -1023,6 +1023,18 @@ def open_spam(self, url):
+             "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"),
+             "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/")
+ 
++    def test_local_file_open(self):
++        class DummyURLopener(urllib.URLopener):
++            def open_local_file(self, url):
++                return url
++        self.assertEqual(DummyURLopener().open(
++            'local-file://example'), '//example')
++        self.assertEqual(DummyURLopener().open(
++            'local_file://example'), '//example')
++        self.assertRaises(IOError, urllib.urlopen,
++            'local-file://example')
++        self.assertRaises(IOError, urllib.urlopen,
++            'local_file://example')
+ 
+ # Just commented them out.
+ # Can't really tell why keep failing in windows and sparc.
+diff --git a/Lib/urllib.py b/Lib/urllib.py
+index d85504a5cb7e..a24e9a5c68fb 100644
+--- a/Lib/urllib.py
++++ b/Lib/urllib.py
+@@ -203,7 +203,10 @@ def open(self, fullurl, data=None):
+         name = 'open_' + urltype
+         self.type = urltype
+         name = name.replace('-', '_')
+-        if not hasattr(self, name):
++        
++        # bpo-35907: # disallow the file reading with the type not allowed
++        if not hasattr(self, name) or \
++            (self == _urlopener and name == 'open_local_file'):
+             if proxy:
+                 return self.open_unknown_proxy(proxy, fullurl, data)
+             else:
+
+From b86392511acd4cd30dc68711fa22f9f93228715a Mon Sep 17 00:00:00 2001
+From: "blurb-it[bot]" <blurb-it[bot]@users.noreply.github.com>
+Date: Wed, 13 Feb 2019 17:21:11 +0000
+Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?=
+ =?UTF-8?q?rb=5Fit.?=
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+---
+ .../NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst | 1 +
+ 1 file changed, 1 insertion(+)
+ create mode 100644 Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst
+
+diff --git a/Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst b/Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst
+new file mode 100644
+index 000000000000..8118a5f40583
+--- /dev/null
++++ b/Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst
+@@ -0,0 +1 @@
++Avoid file reading as disallowing the unnecessary URL scheme in urllib.urlopen
+\ No newline at end of file
+
+From f20a31c7364fecdd3197e0180a5857e23aa15065 Mon Sep 17 00:00:00 2001
+From: SH <push0ebp@gmail.com>
+Date: Fri, 17 May 2019 02:31:18 +0900
+Subject: [PATCH 3/6] Update 2019-02-13-17-21-10.bpo-35907.ckk2zg.rst
+
+Add prefix "CVE-2019-9948: "
+---
+ .../next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst       | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst b/Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst
+index 8118a5f40583..bb187d8d65a5 100644
+--- a/Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst
++++ b/Misc/NEWS.d/next/Library/2019-02-13-17-21-10.bpo-35907.ckk2zg.rst
+@@ -1 +1 @@
+-Avoid file reading as disallowing the unnecessary URL scheme in urllib.urlopen
+\ No newline at end of file
++CVE-2019-9948: Avoid file reading as disallowing the unnecessary URL scheme in urllib.urlopen
+
+From 179a5f75f1121dab271fe8f90eb35145f9dcbbda Mon Sep 17 00:00:00 2001
+From: Sihoon Lee <push0ebp@gmail.com>
+Date: Fri, 17 May 2019 02:41:06 +0900
+Subject: [PATCH 4/6] Update test_urllib.py and urllib.py\nchange assertEqual
+ into assertRasies in DummyURLopener test, and simplify mitigation
+
+---
+ Lib/test/test_urllib.py | 11 +++--------
+ Lib/urllib.py           |  4 ++--
+ 2 files changed, 5 insertions(+), 10 deletions(-)
+
+diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
+index e5f210e62a18..1e23dfb0bb16 100644
+--- a/Lib/test/test_urllib.py
++++ b/Lib/test/test_urllib.py
+@@ -1027,14 +1027,9 @@ def test_local_file_open(self):
+         class DummyURLopener(urllib.URLopener):
+             def open_local_file(self, url):
+                 return url
+-        self.assertEqual(DummyURLopener().open(
+-            'local-file://example'), '//example')
+-        self.assertEqual(DummyURLopener().open(
+-            'local_file://example'), '//example')
+-        self.assertRaises(IOError, urllib.urlopen,
+-            'local-file://example')
+-        self.assertRaises(IOError, urllib.urlopen,
+-            'local_file://example')
++        for url in ('local_file://example', 'local-file://example'):
++            self.assertRaises(IOError, DummyURLopener().open, url)
++            self.assertRaises(IOError, urllib.urlopen, url)
+ 
+ # Just commented them out.
+ # Can't really tell why keep failing in windows and sparc.
+diff --git a/Lib/urllib.py b/Lib/urllib.py
+index a24e9a5c68fb..39b834054e9e 100644
+--- a/Lib/urllib.py
++++ b/Lib/urllib.py
+@@ -203,10 +203,10 @@ def open(self, fullurl, data=None):
+         name = 'open_' + urltype
+         self.type = urltype
+         name = name.replace('-', '_')
+-        
++
+         # bpo-35907: # disallow the file reading with the type not allowed
+         if not hasattr(self, name) or \
+-            (self == _urlopener and name == 'open_local_file'):
++            getattr(self, name) == self.open_local_file:
+             if proxy:
+                 return self.open_unknown_proxy(proxy, fullurl, data)
+             else:
+
+From 3cda03c00109f9c1ae0df1760ecd60915cef105e Mon Sep 17 00:00:00 2001
+From: SH <push0ebp@gmail.com>
+Date: Tue, 21 May 2019 22:21:15 +0900
+Subject: [PATCH 5/6] Update urllib.py
+
+Modify the object to string in check method name.
+---
+ Lib/urllib.py | 3 +--
+ 1 file changed, 1 insertion(+), 2 deletions(-)
+
+diff --git a/Lib/urllib.py b/Lib/urllib.py
+index 39b834054e9e..0bf5f4d5a21b 100644
+--- a/Lib/urllib.py
++++ b/Lib/urllib.py
+@@ -205,8 +205,7 @@ def open(self, fullurl, data=None):
+         name = name.replace('-', '_')
+ 
+         # bpo-35907: # disallow the file reading with the type not allowed
+-        if not hasattr(self, name) or \
+-            getattr(self, name) == self.open_local_file:
++        if not hasattr(self, name) or name == 'open_local_file':
+             if proxy:
+                 return self.open_unknown_proxy(proxy, fullurl, data)
+             else:
+
+From 8b7d7abff8c633e29a8f10bbf9cc7d9e656b0eec Mon Sep 17 00:00:00 2001
+From: SH <push0ebp@gmail.com>
+Date: Wed, 22 May 2019 03:48:56 +0900
+Subject: [PATCH 6/6] Update urllib.py
+
+Fix typo
+---
+ Lib/urllib.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Lib/urllib.py b/Lib/urllib.py
+index 0bf5f4d5a21b..156879dd0a14 100644
+--- a/Lib/urllib.py
++++ b/Lib/urllib.py
+@@ -204,7 +204,7 @@ def open(self, fullurl, data=None):
+         self.type = urltype
+         name = name.replace('-', '_')
+ 
+-        # bpo-35907: # disallow the file reading with the type not allowed
++        # bpo-35907: disallow the file reading with the type not allowed
+         if not hasattr(self, name) or name == 'open_local_file':
+             if proxy:
+                 return self.open_unknown_proxy(proxy, fullurl, data)

lang/python/python3/Makefile

@@ -14,7 +14,7 @@ PYTHON_VERSION:=$(PYTHON3_VERSION)
 PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO)
 
 PKG_NAME:=python3
-PKG_RELEASE:=12
+PKG_RELEASE:=13
 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
 
 PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz

lang/python/python3/patches/023-bpo-35907-CVE-2019-9948-urllib-rejects-local_file-scheme-GH-13505.patch

@@ -0,0 +1,76 @@
+From 3fa72516a390fa8e3552007814e8dc1248686eb5 Mon Sep 17 00:00:00 2001
+From: Victor Stinner <victor.stinner@gmail.com>
+Date: Wed, 22 May 2019 22:15:01 +0200
+Subject: [PATCH] bpo-35907, CVE-2019-9948: urllib rejects local_file:// scheme
+ (GH-13474)
+
+CVE-2019-9948: Avoid file reading as disallowing the unnecessary URL
+scheme in URLopener().open() and URLopener().retrieve()
+of urllib.request.
+
+Co-Authored-By: SH <push0ebp@gmail.com>
+(cherry picked from commit 0c2b6a3943aa7b022e8eb4bfd9bffcddebf9a587)
+---
+ Lib/test/test_urllib.py                        | 18 ++++++++++++++++++
+ Lib/urllib/request.py                          |  2 +-
+ .../2019-05-21-23-20-18.bpo-35907.NC_zNK.rst   |  2 ++
+ 3 files changed, 21 insertions(+), 1 deletion(-)
+ create mode 100644 Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst
+
+diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
+index 7214492eca9d..7ec365b928a5 100644
+--- a/Lib/test/test_urllib.py
++++ b/Lib/test/test_urllib.py
+@@ -16,6 +16,7 @@
+     ssl = None
+ import sys
+ import tempfile
++import warnings
+ from nturl2path import url2pathname, pathname2url
+ 
+ from base64 import b64encode
+@@ -1463,6 +1464,23 @@ def open_spam(self, url):
+                 "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"),
+                 "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/")
+ 
++    def test_local_file_open(self):
++        # bpo-35907, CVE-2019-9948: urllib must reject local_file:// scheme
++        class DummyURLopener(urllib.request.URLopener):
++            def open_local_file(self, url):
++                return url
++
++        with warnings.catch_warnings(record=True):
++            warnings.simplefilter("ignore", DeprecationWarning)
++
++            for url in ('local_file://example', 'local-file://example'):
++                self.assertRaises(OSError, urllib.request.urlopen, url)
++                self.assertRaises(OSError, urllib.request.URLopener().open, url)
++                self.assertRaises(OSError, urllib.request.URLopener().retrieve, url)
++                self.assertRaises(OSError, DummyURLopener().open, url)
++                self.assertRaises(OSError, DummyURLopener().retrieve, url)
++
++
+ # Just commented them out.
+ # Can't really tell why keep failing in windows and sparc.
+ # Everywhere else they work ok, but on those machines, sometimes
+diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
+index d38f725d8e9f..37b254862887 100644
+--- a/Lib/urllib/request.py
++++ b/Lib/urllib/request.py
+@@ -1746,7 +1746,7 @@ def open(self, fullurl, data=None):
+         name = 'open_' + urltype
+         self.type = urltype
+         name = name.replace('-', '_')
+-        if not hasattr(self, name):
++        if not hasattr(self, name) or name == 'open_local_file':
+             if proxy:
+                 return self.open_unknown_proxy(proxy, fullurl, data)
+             else:
+diff --git a/Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst b/Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst
+new file mode 100644
+index 000000000000..16adc7a94e2f
+--- /dev/null
++++ b/Misc/NEWS.d/next/Security/2019-05-21-23-20-18.bpo-35907.NC_zNK.rst
+@@ -0,0 +1,2 @@
++CVE-2019-9948: Avoid file reading as disallowing the unnecessary URL scheme in
++``URLopener().open()`` and ``URLopener().retrieve()`` of :mod:`urllib.request`.