File CVE-2025-4516-DecodeError-handler.patch of Package python3.39333
From 51a69b853783609a51bdbae6c1e20af6a5cc68ce Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Tue, 20 May 2025 15:46:57 +0300
Subject: [PATCH] [CVE-2025-4516] Fix use-after-free in the unicode-escape
decoder with an error handler
Cut disused recode_encoding logic in _PyBytes_DecodeEscape.
All call sites pass NULL for `recode_encoding`, so this path is
completely untested. That's been true since before Python 3.0.
It adds significant complexity to this logic, so it's best to
take it out.
All call sites now have a literal NULL, and that's been true since
commit 768921cf3 eliminated a conditional (`foo ? bar : NULL`) at
the call site in Python/ast.c where we're parsing a bytes literal.
But even before then, that condition `foo` had been a constant
since unadorned string literals started meaning Unicode, in commit
572dbf8f1 aka v3.0a1~1035 .
The `unicode` parameter is already unused, so mark it as unused too.
The code that acted on it was also taken out before Python 3.0, in
commit 8d30cc014 aka v3.0a1~1031 .
The function (PyBytes_DecodeEscape) is exposed in the API, but it's
never been documented.
Fixes: bsc#1243273 (CVE-2025-4516)
Fixes: gh#python/cpython#133767
From-PR: gh#python/cpython!134346
Patch: CVE-2025-4516-DecodeError-handler.patch
---
Include/longobject.h | 2
Include/modsupport.h | 8
Include/unicodeobject.h | 31 +
Lib/encodings/unicode_escape.py | 9
Lib/test/test_codeccallbacks.py | 36 +
Lib/test/test_codecs.py | 83 +++
Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst | 2
Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst | 2
Modules/_codecsmodule.c | 30 -
Modules/clinic/_codecsmodule.c.h | 28 -
Objects/bytesobject.c | 118 +----
Objects/unicodeobject.c | 234 ++++++++--
Python/ast.c | 2
13 files changed, 436 insertions(+), 149 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst
create mode 100644 Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst
--- a/Include/longobject.h
+++ b/Include/longobject.h
@@ -66,7 +66,7 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(vo
#endif /* SIZEOF_VOID_P */
/* Used by Python/mystrtoul.c, _PyBytes_FromHex(),
- _PyBytes_DecodeEscapeRecode(), etc. */
+ _PyBytes_DecodeEscape(), etc. */
#ifndef Py_LIMITED_API
PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
#endif
--- a/Include/modsupport.h
+++ b/Include/modsupport.h
@@ -41,6 +41,14 @@ PyAPI_FUNC(PyObject *) Py_BuildValue(con
PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...);
#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyArg_UnpackStack(
+ PyObject *const *args,
+ Py_ssize_t nargs,
+ const char *name,
+ Py_ssize_t min,
+ Py_ssize_t max,
+ ...);
+
PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kw);
PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args);
#endif
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -1505,12 +1505,33 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeUn
);
#ifndef Py_LIMITED_API
+/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */
+PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful(
+ const char *string, /* Unicode-Escape encoded string */
+ Py_ssize_t length, /* size of string */
+ const char *errors, /* error handling */
+ Py_ssize_t *consumed /* bytes consumed */
+);
/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape
chars. */
-PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape(
+PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal2(
+ const char *string, /* Unicode-Escape encoded string */
+ Py_ssize_t length, /* size of string */
+ const char *errors, /* error handling */
+ Py_ssize_t *consumed, /* bytes consumed */
+ int *first_invalid_escape_char, /* on return, if not -1, contain the first
+ invalid escaped char (<= 0xff) or invalid
+ octal escape (> 0xff) in string. */
+ const char **first_invalid_escape_ptr); /* on return, if not NULL, may
+ point to the first invalid escaped
+ char in string.
+ May be NULL if errors is not NULL. */
+// Export for binary compatibility.
+PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal(
const char *string, /* Unicode-Escape encoded string */
Py_ssize_t length, /* size of string */
const char *errors, /* error handling */
+ Py_ssize_t *consumed, /* bytes consumed */
const char **first_invalid_escape /* on return, points to first
invalid escaped char in
string. */
@@ -1547,6 +1568,14 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeRa
);
#endif
+/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */
+PyAPI_FUNC(PyObject*) _PyUnicode_DecodeRawUnicodeEscapeStateful(
+ const char *string, /* Unicode-Escape encoded string */
+ Py_ssize_t length, /* size of string */
+ const char *errors, /* error handling */
+ Py_ssize_t *consumed /* bytes consumed */
+);
+
/* --- Unicode Internal Codec ---------------------------------------------
Only for internal use in _codecsmodule.c */
--- a/Lib/encodings/unicode_escape.py
+++ b/Lib/encodings/unicode_escape.py
@@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.Incremen
def encode(self, input, final=False):
return codecs.unicode_escape_encode(input, self.errors)[0]
-class IncrementalDecoder(codecs.IncrementalDecoder):
- def decode(self, input, final=False):
- return codecs.unicode_escape_decode(input, self.errors)[0]
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+ def _buffer_decode(self, input, errors, final):
+ return codecs.unicode_escape_decode(input, errors, final)
class StreamWriter(Codec,codecs.StreamWriter):
pass
class StreamReader(Codec,codecs.StreamReader):
- pass
+ def decode(self, input, errors='strict'):
+ return codecs.unicode_escape_decode(input, errors, False)
### encodings module API
--- a/Lib/test/test_codeccallbacks.py
+++ b/Lib/test/test_codeccallbacks.py
@@ -1004,7 +1004,7 @@ class CodecCallbackTest(unittest.TestCas
text = 'abc<def>ghi'*n
text.translate(charmap)
- def test_mutatingdecodehandler(self):
+ def test_mutating_decode_handler(self):
baddata = [
("ascii", b"\xff"),
("utf-7", b"++"),
@@ -1044,6 +1044,40 @@ class CodecCallbackTest(unittest.TestCas
for (encoding, data) in baddata:
self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242")
+ def test_mutating_decode_handler_unicode_escape(self):
+ decode = codecs.unicode_escape_decode
+ def mutating(exc):
+ if isinstance(exc, UnicodeDecodeError):
+ r = data.get(exc.object[:exc.end])
+ if r is not None:
+ exc.object = r[0] + exc.object[exc.end:]
+ return ('\u0404', r[1])
+ raise AssertionError("don't know how to handle %r" % exc)
+
+ codecs.register_error('test.mutating2', mutating)
+ data = {
+ br'\x0': (b'\\', 0),
+ br'\x3': (b'xxx\\', 3),
+ br'\x5': (b'x\\', 1),
+ }
+ def check(input, expected, msg):
+ with self.assertWarns(DeprecationWarning) as cm:
+ self.assertEqual(decode(input, 'test.mutating2'), (expected, len(input)))
+ self.assertIn(msg, str(cm.warning))
+
+ check(br'\x0n\z', '\u0404\n\\z', r"invalid escape sequence '\z'")
+ check(br'\x0z', '\u0404\\z', r"invalid escape sequence '\z'")
+
+ check(br'\x3n\zr', '\u0404\n\\zr', r"invalid escape sequence '\z'")
+ check(br'\x3zr', '\u0404\\zr', r"invalid escape sequence '\z'")
+ check(br'\x3z5', '\u0404\\z5', r"invalid escape sequence '\z'")
+ check(memoryview(br'\x3z5x')[:-1], '\u0404\\z5', r"invalid escape sequence '\z'")
+ check(memoryview(br'\x3z5xy')[:-2], '\u0404\\z5', r"invalid escape sequence '\z'")
+
+ check(br'\x5n\z', '\u0404\n\\z', r"invalid escape sequence '\z'")
+ check(br'\x5z', '\u0404\\z', r"invalid escape sequence '\z'")
+ check(memoryview(br'\x5zy')[:-1], '\u0404\\z', r"invalid escape sequence '\z'")
+
# issue32583
def test_crashing_decode_handler(self):
# better generating one more character to fill the extra space slot
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1206,20 +1206,32 @@ class EscapeDecodeTest(unittest.TestCase
check(br"[\501]", b"[A]")
check(br"[\x41]", b"[A]")
check(br"[\x410]", b"[A0]")
+
+ def test_warnings(self):
+ decode = codecs.escape_decode
+ check = coding_checker(self, decode)
for i in range(97, 123):
b = bytes([i])
if b not in b'abfnrtvx':
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\%c'" % i):
check(b"\\" + b, b"\\" + b)
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\%c'" % (i-32)):
check(b"\\" + b.upper(), b"\\" + b.upper())
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\8'"):
check(br"\8", b"\\8")
with self.assertWarns(DeprecationWarning):
check(br"\9", b"\\9")
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\\xfa'") as cm:
check(b"\\\xfa", b"\\\xfa")
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\z'"):
+ self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4))
+
def test_errors(self):
decode = codecs.escape_decode
self.assertRaises(ValueError, decode, br"\x")
@@ -2428,7 +2440,11 @@ class TypesTest(unittest.TestCase):
(r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
-class UnicodeEscapeTest(unittest.TestCase):
+class UnicodeEscapeTest(ReadTest, unittest.TestCase):
+ encoding = "unicode-escape"
+
+ test_lone_surrogates = None
+
def test_empty(self):
self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
@@ -2484,20 +2500,31 @@ class UnicodeEscapeTest(unittest.TestCas
check(br"[\x410]", "[A0]")
check(br"\u20ac", "\u20ac")
check(br"\U0001d120", "\U0001d120")
+
+ def test_decode_warnings(self):
+ decode = codecs.unicode_escape_decode
+ check = coding_checker(self, decode)
for i in range(97, 123):
b = bytes([i])
if b not in b'abfnrtuvx':
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\%c'" % i):
check(b"\\" + b, "\\" + chr(i))
if b.upper() not in b'UN':
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\%c'" % (i-32)):
check(b"\\" + b.upper(), "\\" + chr(i-32))
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\8'"):
check(br"\8", "\\8")
with self.assertWarns(DeprecationWarning):
check(br"\9", "\\9")
- with self.assertWarns(DeprecationWarning):
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\\xfa'") as cm:
check(b"\\\xfa", "\\\xfa")
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"invalid escape sequence '\\z'"):
+ self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4))
def test_decode_errors(self):
decode = codecs.unicode_escape_decode
@@ -2515,6 +2542,44 @@ class UnicodeEscapeTest(unittest.TestCas
self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
+ def test_partial(self):
+ self.check_partial(
+ "\x00\t\n\r\\\xff\uffff\U00010000",
+ [
+ '',
+ '',
+ '',
+ '\x00',
+ '\x00',
+ '\x00\t',
+ '\x00\t',
+ '\x00\t\n',
+ '\x00\t\n',
+ '\x00\t\n\r',
+ '\x00\t\n\r',
+ '\x00\t\n\r\\',
+ '\x00\t\n\r\\',
+ '\x00\t\n\r\\',
+ '\x00\t\n\r\\',
+ '\x00\t\n\r\\\xff',
+ '\x00\t\n\r\\\xff',
+ '\x00\t\n\r\\\xff',
+ '\x00\t\n\r\\\xff',
+ '\x00\t\n\r\\\xff',
+ '\x00\t\n\r\\\xff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff',
+ '\x00\t\n\r\\\xff\uffff\U00010000',
+ ]
+ )
class RawUnicodeEscapeTest(unittest.TestCase):
def test_empty(self):
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst
@@ -0,0 +1,2 @@
+Fix incremental decoder and stream reader in the "unicode-escape" codec.
+Previously they failed if the escape sequence was split.
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst
@@ -0,0 +1,2 @@
+Fix use-after-free in the "unicode-escape" decoder with a non-"strict" error
+handler.
--- a/Modules/_codecsmodule.c
+++ b/Modules/_codecsmodule.c
@@ -519,35 +519,41 @@ _codecs_utf_32_ex_decode_impl(PyObject *
/*[clinic input]
_codecs.unicode_escape_decode
data: Py_buffer(accept={str, buffer})
- errors: str(accept={str, NoneType}) = NULL
+ errors: str(accept={str, NoneType}) = None
+ final: bool = True
/
[clinic start generated code]*/
static PyObject *
_codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
- const char *errors)
-/*[clinic end generated code: output=3ca3c917176b82ab input=49fd27d06813a7f5]*/
+ const char *errors, int final)
+/*[clinic end generated code: output=b284f97b12c635ee input=15019f081ffe272b]*/
{
- PyObject *decoded = PyUnicode_DecodeUnicodeEscape(data->buf, data->len,
- errors);
- return codec_tuple(decoded, data->len);
+ Py_ssize_t consumed = data->len;
+ PyObject *decoded = _PyUnicode_DecodeUnicodeEscapeStateful(data->buf, data->len,
+ errors,
+ final ? NULL : &consumed);
+ return codec_tuple(decoded, consumed);
}
/*[clinic input]
_codecs.raw_unicode_escape_decode
data: Py_buffer(accept={str, buffer})
- errors: str(accept={str, NoneType}) = NULL
+ errors: str(accept={str, NoneType}) = None
+ final: bool = True
/
[clinic start generated code]*/
static PyObject *
_codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
- const char *errors)
-/*[clinic end generated code: output=c98eeb56028070a6 input=770903a211434ebc]*/
+ const char *errors, int final)
+/*[clinic end generated code: output=11dbd96301e2879e input=b93f823aa8c343ad]*/
{
- PyObject *decoded = PyUnicode_DecodeRawUnicodeEscape(data->buf, data->len,
- errors);
- return codec_tuple(decoded, data->len);
+ Py_ssize_t consumed = data->len;
+ PyObject *decoded = _PyUnicode_DecodeRawUnicodeEscapeStateful(data->buf, data->len,
+ errors,
+ final ? NULL : &consumed);
+ return codec_tuple(decoded, consumed);
}
/*[clinic input]
--- a/Modules/clinic/_codecsmodule.c.h
+++ b/Modules/clinic/_codecsmodule.c.h
@@ -594,7 +594,7 @@ exit:
}
PyDoc_STRVAR(_codecs_unicode_escape_decode__doc__,
-"unicode_escape_decode($module, data, errors=None, /)\n"
+"unicode_escape_decode($module, data, errors=None, final=True, /)\n"
"--\n"
"\n");
@@ -603,20 +603,21 @@ PyDoc_STRVAR(_codecs_unicode_escape_deco
static PyObject *
_codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
- const char *errors);
+ const char *errors, int final);
static PyObject *
_codecs_unicode_escape_decode(PyObject *module, PyObject *args)
{
PyObject *return_value = NULL;
- Py_buffer data = {NULL, NULL};
+ Py_buffer data = {0};
const char *errors = NULL;
+ int final = 1;
- if (!PyArg_ParseTuple(args, "s*|z:unicode_escape_decode",
- &data, &errors)) {
+ if (!PyArg_ParseTuple(args, "s*|zp:unicode_escape_decode",
+ &data, &errors, &final)) {
goto exit;
}
- return_value = _codecs_unicode_escape_decode_impl(module, &data, errors);
+ return_value = _codecs_unicode_escape_decode_impl(module, &data, errors, final);
exit:
/* Cleanup for data */
@@ -628,7 +629,7 @@ exit:
}
PyDoc_STRVAR(_codecs_raw_unicode_escape_decode__doc__,
-"raw_unicode_escape_decode($module, data, errors=None, /)\n"
+"raw_unicode_escape_decode($module, data, errors=None, final=True, /)\n"
"--\n"
"\n");
@@ -637,20 +638,21 @@ PyDoc_STRVAR(_codecs_raw_unicode_escape_
static PyObject *
_codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
- const char *errors);
+ const char *errors, int final);
static PyObject *
_codecs_raw_unicode_escape_decode(PyObject *module, PyObject *args)
{
PyObject *return_value = NULL;
- Py_buffer data = {NULL, NULL};
+ Py_buffer data = {0};
const char *errors = NULL;
+ int final = 1;
- if (!PyArg_ParseTuple(args, "s*|z:raw_unicode_escape_decode",
- &data, &errors)) {
+ if (!PyArg_ParseTuple(args, "s*|zp:raw_unicode_escape_decode",
+ &data, &errors, &final)) {
goto exit;
}
- return_value = _codecs_raw_unicode_escape_decode_impl(module, &data, errors);
+ return_value = _codecs_raw_unicode_escape_decode_impl(module, &data, errors, final);
exit:
/* Cleanup for data */
@@ -1536,4 +1538,4 @@ exit:
#ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF
#define _CODECS_CODE_PAGE_ENCODE_METHODDEF
#endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */
-/*[clinic end generated code: output=6d6afcabde10ed79 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=304d4afdb776a218 input=a9049054013a1b77]*/
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -1068,61 +1068,12 @@ _PyBytes_FormatEx(const char *format, Py
return NULL;
}
-/* =-= */
-
-static void
-bytes_dealloc(PyObject *op)
-{
- Py_TYPE(op)->tp_free(op);
-}
-
-/* Unescape a backslash-escaped string. If unicode is non-zero,
- the string is a u-literal. If recode_encoding is non-zero,
- the string is UTF-8 encoded and should be re-encoded in the
- specified encoding. */
-
-static char *
-_PyBytes_DecodeEscapeRecode(const char **s, const char *end,
- const char *errors, const char *recode_encoding,
- _PyBytesWriter *writer, char *p)
-{
- PyObject *u, *w;
- const char* t;
-
- t = *s;
- /* Decode non-ASCII bytes as UTF-8. */
- while (t < end && (*t & 0x80))
- t++;
- u = PyUnicode_DecodeUTF8(*s, t - *s, errors);
- if (u == NULL)
- return NULL;
-
- /* Recode them in target encoding. */
- w = PyUnicode_AsEncodedString(u, recode_encoding, errors);
- Py_DECREF(u);
- if (w == NULL)
- return NULL;
- assert(PyBytes_Check(w));
-
- /* Append bytes to output buffer. */
- writer->min_size--; /* subtract 1 preallocated byte */
- p = _PyBytesWriter_WriteBytes(writer, p,
- PyBytes_AS_STRING(w),
- PyBytes_GET_SIZE(w));
- Py_DECREF(w);
- if (p == NULL)
- return NULL;
-
- *s = t;
- return p;
-}
-
-PyObject *_PyBytes_DecodeEscape(const char *s,
+/* Unescape a backslash-escaped string. */
+PyObject *_PyBytes_DecodeEscape2(const char *s,
Py_ssize_t len,
const char *errors,
- Py_ssize_t unicode,
- const char *recode_encoding,
- const char **first_invalid_escape)
+ int *first_invalid_escape_char,
+ const char **first_invalid_escape_ptr)
{
int c;
char *p;
@@ -1136,23 +1087,13 @@ PyObject *_PyBytes_DecodeEscape(const ch
return NULL;
writer.overallocate = 1;
- *first_invalid_escape = NULL;
+ *first_invalid_escape_char = -1;
+ *first_invalid_escape_ptr = NULL;
end = s + len;
while (s < end) {
if (*s != '\\') {
- non_esc:
- if (!(recode_encoding && (*s & 0x80))) {
- *p++ = *s++;
- }
- else {
- /* non-ASCII character and need to recode */
- p = _PyBytes_DecodeEscapeRecode(&s, end,
- errors, recode_encoding,
- &writer, p);
- if (p == NULL)
- goto failed;
- }
+ *p++ = *s++;
continue;
}
@@ -1222,14 +1163,13 @@ PyObject *_PyBytes_DecodeEscape(const ch
break;
default:
- if (*first_invalid_escape == NULL) {
- *first_invalid_escape = s-1; /* Back up one char, since we've
- already incremented s. */
+ if (*first_invalid_escape_char == -1) {
+ *first_invalid_escape_char = (unsigned char)s[-1];
+ /* Back up one char, since we've already incremented s. */
+ *first_invalid_escape_ptr = s - 1;
}
*p++ = '\\';
s--;
- goto non_esc; /* an arbitrary number of unescaped
- UTF-8 bytes may follow. */
}
}
@@ -1240,22 +1180,38 @@ PyObject *_PyBytes_DecodeEscape(const ch
return NULL;
}
-PyObject *PyBytes_DecodeEscape(const char *s,
+// Export for binary compatibility.
+PyObject *_PyBytes_DecodeEscape(const char *s,
Py_ssize_t len,
const char *errors,
Py_ssize_t unicode,
- const char *recode_encoding)
+ const char *recode_encoding,
+ const char **first_invalid_escape)
+{
+ int first_invalid_escape_char;
+ return _PyBytes_DecodeEscape2(
+ s, len, errors,
+ &first_invalid_escape_char,
+ first_invalid_escape);
+}
+
+PyObject *PyBytes_DecodeEscape(const char *s,
+ Py_ssize_t len,
+ const char *errors,
+ Py_ssize_t Py_UNUSED(unicode),
+ const char *Py_UNUSED(recode_encoding))
{
- const char* first_invalid_escape;
- PyObject *result = _PyBytes_DecodeEscape(s, len, errors, unicode,
- recode_encoding,
- &first_invalid_escape);
+ int first_invalid_escape_char;
+ const char *first_invalid_escape_ptr;
+ PyObject *result = _PyBytes_DecodeEscape2(s, len, errors,
+ &first_invalid_escape_char,
+ &first_invalid_escape_ptr);
if (result == NULL)
return NULL;
- if (first_invalid_escape != NULL) {
+ if (first_invalid_escape_char != -1) {
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
"invalid escape sequence '\\%c'",
- (unsigned char)*first_invalid_escape) < 0) {
+ first_invalid_escape_char) < 0) {
Py_DECREF(result);
return NULL;
}
@@ -2876,8 +2832,8 @@ PyTypeObject PyBytes_Type = {
"bytes",
PyBytesObject_SIZE,
sizeof(char),
- bytes_dealloc, /* tp_dealloc */
- 0, /* tp_print */
+ 0, /* tp_dealloc */
+ 0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -5998,21 +5998,28 @@ PyUnicode_AsUTF16String(PyObject *unicod
static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
PyObject *
-_PyUnicode_DecodeUnicodeEscape(const char *s,
+_PyUnicode_DecodeUnicodeEscapeInternal2(const char *s,
Py_ssize_t size,
const char *errors,
- const char **first_invalid_escape)
+ Py_ssize_t *consumed,
+ int *first_invalid_escape_char,
+ const char **first_invalid_escape_ptr)
{
const char *starts = s;
+ const char *initial_starts = starts;
_PyUnicodeWriter writer;
const char *end;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
// so we can remember if we've seen an invalid escape char or not
- *first_invalid_escape = NULL;
+ *first_invalid_escape_char = -1;
+ *first_invalid_escape_ptr = NULL;
if (size == 0) {
+ if (consumed) {
+ *consumed = 0;
+ }
_Py_RETURN_UNICODE_EMPTY();
}
/* Escaped strings will always be longer than the resulting
@@ -6031,8 +6038,6 @@ _PyUnicode_DecodeUnicodeEscape(const cha
unsigned char c = (unsigned char) *s++;
Py_UCS4 ch;
int count;
- Py_ssize_t startinpos;
- Py_ssize_t endinpos;
const char *message;
#define WRITE_ASCII_CHAR(ch) \
@@ -6059,11 +6064,11 @@ _PyUnicode_DecodeUnicodeEscape(const cha
continue;
}
- startinpos = s - starts - 1;
+ Py_ssize_t startinpos = s - starts - 1;
/* \ - Escapes */
if (s >= end) {
message = "\\ at end of string";
- goto error;
+ goto incomplete;
}
c = (unsigned char) *s++;
@@ -6117,7 +6122,10 @@ _PyUnicode_DecodeUnicodeEscape(const cha
count = 8;
message = "truncated \\UXXXXXXXX escape";
hexescape:
- for (ch = 0; count && s < end; ++s, --count) {
+ for (ch = 0; count; ++s, --count) {
+ if (s >= end) {
+ goto incomplete;
+ }
c = (unsigned char)*s;
ch <<= 4;
if (c >= '0' && c <= '9') {
@@ -6130,12 +6138,9 @@ _PyUnicode_DecodeUnicodeEscape(const cha
ch += c - ('A' - 10);
}
else {
- break;
+ goto error;
}
}
- if (count) {
- goto error;
- }
/* when we get here, ch is a 32-bit unicode character */
if (ch > MAX_UNICODE) {
@@ -6162,14 +6167,20 @@ _PyUnicode_DecodeUnicodeEscape(const cha
}
message = "malformed \\N character escape";
- if (s < end && *s == '{') {
+ if (s >= end) {
+ goto incomplete;
+ }
+ if (*s == '{') {
const char *start = ++s;
size_t namelen;
/* look for the closing brace */
while (s < end && *s != '}')
s++;
+ if (s >= end) {
+ goto incomplete;
+ }
namelen = s - start;
- if (namelen && s < end) {
+ if (namelen) {
/* found a name. look it up in the unicode database */
s++;
ch = 0xffffffff; /* in case 'getcode' messes up */
@@ -6186,17 +6197,25 @@ _PyUnicode_DecodeUnicodeEscape(const cha
goto error;
default:
- if (*first_invalid_escape == NULL) {
- *first_invalid_escape = s-1; /* Back up one char, since we've
- already incremented s. */
+ if (*first_invalid_escape_char == -1) {
+ *first_invalid_escape_char = c;
+ if (starts == initial_starts) {
+ /* Back up one char, since we've already incremented s. */
+ *first_invalid_escape_ptr = s - 1;
+ }
}
WRITE_ASCII_CHAR('\\');
WRITE_CHAR(c);
continue;
}
- error:
- endinpos = s-starts;
+ incomplete:
+ if (consumed) {
+ *consumed = startinpos;
+ break;
+ }
+ error:;
+ Py_ssize_t endinpos = s-starts;
writer.min_length = end - s + writer.pos;
if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
@@ -6222,20 +6241,39 @@ _PyUnicode_DecodeUnicodeEscape(const cha
return NULL;
}
+// Export for binary compatibility.
PyObject *
-PyUnicode_DecodeUnicodeEscape(const char *s,
+_PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
+ Py_ssize_t size,
+ const char *errors,
+ Py_ssize_t *consumed,
+ const char **first_invalid_escape)
+{
+ int first_invalid_escape_char;
+ return _PyUnicode_DecodeUnicodeEscapeInternal2(
+ s, size, errors, consumed,
+ &first_invalid_escape_char,
+ first_invalid_escape);
+}
+
+PyObject *
+_PyUnicode_DecodeUnicodeEscapeStateful(const char *s,
Py_ssize_t size,
- const char *errors)
+ const char *errors,
+ Py_ssize_t *consumed)
{
- const char *first_invalid_escape;
- PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors,
- &first_invalid_escape);
+ int first_invalid_escape_char;
+ const char *first_invalid_escape_ptr;
+ PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal2(s, size, errors,
+ consumed,
+ &first_invalid_escape_char,
+ &first_invalid_escape_ptr);
if (result == NULL)
return NULL;
- if (first_invalid_escape != NULL) {
+ if (first_invalid_escape_char != -1) {
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
"invalid escape sequence '\\%c'",
- (unsigned char)*first_invalid_escape) < 0) {
+ first_invalid_escape_char) < 0) {
Py_DECREF(result);
return NULL;
}
@@ -6243,6 +6281,14 @@ PyUnicode_DecodeUnicodeEscape(const char
return result;
}
+PyObject *
+PyUnicode_DecodeUnicodeEscape(const char *s,
+ Py_ssize_t size,
+ const char *errors)
+{
+ return _PyUnicode_DecodeUnicodeEscapeStateful(s, size, errors, NULL);
+}
+
/* Return a Unicode-Escape string version of the Unicode object. */
PyObject *
@@ -6381,6 +6427,142 @@ PyUnicode_EncodeUnicodeEscape(const Py_U
/* --- Raw Unicode Escape Codec ------------------------------------------- */
PyObject *
+_PyUnicode_DecodeRawUnicodeEscapeStateful(const char *s,
+ Py_ssize_t size,
+ const char *errors,
+ Py_ssize_t *consumed)
+{
+ const char *starts = s;
+ _PyUnicodeWriter writer;
+ const char *end;
+ PyObject *errorHandler = NULL;
+ PyObject *exc = NULL;
+
+ if (size == 0) {
+ if (consumed) {
+ *consumed = 0;
+ }
+ _Py_RETURN_UNICODE_EMPTY();
+ }
+
+ /* Escaped strings will always be longer than the resulting
+ Unicode string, so we start with size here and then reduce the
+ length after conversion to the true value. (But decoding error
+ handler might have to resize the string) */
+ _PyUnicodeWriter_Init(&writer);
+ writer.min_length = size;
+ if (_PyUnicodeWriter_Prepare(&writer, size, 127) < 0) {
+ goto onError;
+ }
+
+ end = s + size;
+ while (s < end) {
+ unsigned char c = (unsigned char) *s++;
+ Py_UCS4 ch;
+ int count;
+ const char *message;
+
+#define WRITE_CHAR(ch) \
+ do { \
+ if (ch <= writer.maxchar) { \
+ assert(writer.pos < writer.size); \
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \
+ } \
+ else if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) { \
+ goto onError; \
+ } \
+ } while(0)
+
+ /* Non-escape characters are interpreted as Unicode ordinals */
+ if (c != '\\' || (s >= end && !consumed)) {
+ WRITE_CHAR(c);
+ continue;
+ }
+
+ Py_ssize_t startinpos = s - starts - 1;
+ /* \ - Escapes */
+ if (s >= end) {
+ assert(consumed);
+ // Set message to silent compiler warning.
+ // Actually it is never used.
+ message = "\\ at end of string";
+ goto incomplete;
+ }
+
+ c = (unsigned char) *s++;
+ if (c == 'u') {
+ count = 4;
+ message = "truncated \\uXXXX escape";
+ }
+ else if (c == 'U') {
+ count = 8;
+ message = "truncated \\UXXXXXXXX escape";
+ }
+ else {
+ assert(writer.pos < writer.size);
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, '\\');
+ WRITE_CHAR(c);
+ continue;
+ }
+
+ /* \uHHHH with 4 hex digits, \U00HHHHHH with 8 */
+ for (ch = 0; count; ++s, --count) {
+ if (s >= end) {
+ goto incomplete;
+ }
+ c = (unsigned char)*s;
+ ch <<= 4;
+ if (c >= '0' && c <= '9') {
+ ch += c - '0';
+ }
+ else if (c >= 'a' && c <= 'f') {
+ ch += c - ('a' - 10);
+ }
+ else if (c >= 'A' && c <= 'F') {
+ ch += c - ('A' - 10);
+ }
+ else {
+ goto error;
+ }
+ }
+ if (ch > MAX_UNICODE) {
+ message = "\\Uxxxxxxxx out of range";
+ goto error;
+ }
+ WRITE_CHAR(ch);
+ continue;
+
+ incomplete:
+ if (consumed) {
+ *consumed = startinpos;
+ break;
+ }
+ error:;
+ Py_ssize_t endinpos = s-starts;
+ writer.min_length = end - s + writer.pos;
+ if (unicode_decode_call_errorhandler_writer(
+ errors, &errorHandler,
+ "rawunicodeescape", message,
+ &starts, &end, &startinpos, &endinpos, &exc, &s,
+ &writer)) {
+ goto onError;
+ }
+ assert(end - s <= writer.size - writer.pos);
+
+#undef WRITE_CHAR
+ }
+ Py_XDECREF(errorHandler);
+ Py_XDECREF(exc);
+ return _PyUnicodeWriter_Finish(&writer);
+
+ onError:
+ _PyUnicodeWriter_Dealloc(&writer);
+ Py_XDECREF(errorHandler);
+ Py_XDECREF(exc);
+ return NULL;
+}
+
+PyObject *
PyUnicode_DecodeRawUnicodeEscape(const char *s,
Py_ssize_t size,
const char *errors)
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -4244,7 +4244,7 @@ decode_unicode_with_escapes(struct compi
s = buf;
const char *first_invalid_escape;
- v = _PyUnicode_DecodeUnicodeEscape(s, len, NULL, &first_invalid_escape);
+ v = _PyUnicode_DecodeUnicodeEscapeInternal(s, len, NULL, NULL, &first_invalid_escape);
if (v != NULL && first_invalid_escape != NULL) {
if (warn_invalid_escape_sequence(c, n, *first_invalid_escape) < 0) {
