pypy/interpreter/test/test_unicodehelper.py::test_latin1_shortcut_bug
space = StdObjSpace
def test_latin1_shortcut_bug(space):
state = space.fromcache(CodecState)
handler = state.encode_error_handler
sin = u"a\xac\u1234\u20ac\u8000"
sin_utf8 = sin.encode("utf-8")
> assert utf8_encode_latin_1(space, sin_utf8, space.newtext(sin_utf8), "backslashreplace", handler) == sin.encode("latin-1", "backslashreplace")
interpreter/test/test_unicodehelper.py:92:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
objspace/std/objspace.py:413: in newtext
return self._newtext_memo(s)
objspace/std/objspace.py:432: in _newtext_memo
return W_UnicodeObject(s, lgt)
<609-codegen /Users/matti/build-worker-x86_64/own-macos-x86-64/build/rpython/tool/sourcetools.py:318>:5: in __init__
return __init___original(self, utf8str, length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = W_UnicodeObject('a\xc2\xac\xe1\x88\xb4\xe2\x82\xac\xe8\x80\x80')
utf8str = 'a\xc2\xac\xe1\x88\xb4\xe2\x82\xac\xe8\x80\x80', length = 12
@enforceargs(utf8str=str)
def __init__(self, utf8str, length):
assert isinstance(utf8str, bytes)
# TODO: how to handle surrogates
assert length >= 0
self._utf8 = utf8str
self._length = length
self._index_storage = rutf8.null_storage()
if CHECK_ALL_STRINGS or not we_are_translated():
# utf8str must always be a valid utf8 string, except maybe with
# explicit surrogate characters---which .decode('utf-8') doesn't
# special-case in Python 2, which is exactly what we want here
try:
if sys.maxunicode == 0xffff:
# can't use .decode('utf-8') because it will add surrogates
real_length = rutf8.check_utf8(utf8str, True)
else:
real_length = len(utf8str.decode('utf-8'))
except (rutf8.CheckError, UnicodeDecodeError):
real_length = -999
if length != real_length:
from rpython.rlib.debug import debug_print
debug_print("!!! BAD UTF8 !!!")
debug_print(str([ord(c) for c in utf8str]))
debug_print("length", length, "real_length", real_length)
> raise BadUtf8
E BadUtf8
objspace/std/unicodeobject.py:86: BadUtf8
---------- Captured stderr call ----------
!!! BAD UTF8 !!!
[97, 194, 172, 225, 136, 180, 226, 130, 172, 232, 128, 128]
length 12 real_length 5
builder: own-macos-x86-64 build #1268+
test: pypy/interpreter/test/test_unicodehelper.py::test_latin1_shortcut_bug