pypy/interpreter/test/test_unicodehelper.py::test_encode_utf_8_combine_surrogates

space = StdObjSpace

    def test_encode_utf_8_combine_surrogates(space):
        """
        In the case of a surrogate pair, the error handler should
        called with a start and stop position of the full surrogate
        pair (new behavior in python3.6)
        """
        #               /--surrogate pair--\
        #    \udc80      \ud800      \udfff
        b = "\xed\xb2\x80\xed\xa0\x80\xed\xbf\xbf"
        _space = FakeSpace(space)
    
        calls = []
    
        def errorhandler(errors, encoding, msg, w_s, start, end):
            """
            This handler will be called twice, so asserting both times:
    
            1. the first time, 0xDC80 will be handled as a single surrogate,
               since it is a standalone character and an invalid surrogate.
            2. the second time, the characters will be 0xD800 and 0xDFFF, since
               that is a valid surrogate pair.
            """
            s = w_s._utf8
            calls.append(s.decode("utf-8")[start:end])
            return 'abc', end, 'b', s, w_s
>       w_b = space.newtext(b)

interpreter/test/test_unicodehelper.py:57: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
objspace/std/objspace.py:413: in newtext
    return self._newtext_memo(s)
objspace/std/objspace.py:432: in _newtext_memo
    return W_UnicodeObject(s, lgt)
<609-codegen /Users/matti/build-worker-x86_64/own-macos-x86-64/build/rpython/tool/sourcetools.py:318>:5: in __init__
    return __init___original(self, utf8str, length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = W_UnicodeObject('\xed\xb2\x80\xed\xa0\x80\xed\xbf\xbf')
utf8str = '\xed\xb2\x80\xed\xa0\x80\xed\xbf\xbf', length = 9

    @enforceargs(utf8str=str)
    def __init__(self, utf8str, length):
        assert isinstance(utf8str, bytes)
        # TODO: how to handle surrogates
        assert length >= 0
        self._utf8 = utf8str
        self._length = length
        self._index_storage = rutf8.null_storage()
        if CHECK_ALL_STRINGS or not we_are_translated():
            # utf8str must always be a valid utf8 string, except maybe with
            # explicit surrogate characters---which .decode('utf-8') doesn't
            # special-case in Python 2, which is exactly what we want here
            try:
                if sys.maxunicode == 0xffff:
                    # can't use .decode('utf-8') because it will add surrogates
                    real_length = rutf8.check_utf8(utf8str, True)
                else:
                    real_length = len(utf8str.decode('utf-8'))
            except (rutf8.CheckError, UnicodeDecodeError):
                real_length = -999
            if length != real_length:
                from rpython.rlib.debug import debug_print
                debug_print("!!! BAD UTF8 !!!")
                debug_print(str([ord(c) for c in utf8str]))
                debug_print("length", length, "real_length", real_length)
>               raise BadUtf8
E               BadUtf8

objspace/std/unicodeobject.py:86: BadUtf8
---------- Captured stderr call ----------
!!! BAD UTF8 !!! 
[237, 178, 128, 237, 160, 128, 237, 191, 191] 
length 9 real_length 3 

builder: own-macos-x86-64 build #1268+

test: pypy/interpreter/test/test_unicodehelper.py::test_encode_utf_8_combine_surrogates