Index: python-cjson-1.0.5/cjson.c =================================================================== --- python-cjson-1.0.5.orig/cjson.c 2010-08-16 08:38:13.532910138 -0400 +++ python-cjson-1.0.5/cjson.c 2010-08-16 08:46:37.012909622 -0400 @@ -11,6 +11,7 @@ #include #include #include +#include "ucnhash.h" typedef struct JSONData { char *str; // the actual json string @@ -54,6 +55,10 @@ typedef int Py_ssize_t; #define True 1 #define False 0 +#ifndef Py_MEMCPY +#define Py_MEMCPY memcpy +#endif + #ifndef INFINITY #define INFINITY HUGE_VAL #endif @@ -112,6 +117,473 @@ decode_bool(JSONData *jsondata) } } +PyObject *_DecodeEscape(const char *s, + Py_ssize_t len, + const char *errors, + Py_ssize_t unicode, + const char *recode_encoding) +{ + int c; + char *p, *buf; + const char *end; + PyObject *v; + Py_ssize_t newlen = recode_encoding ? 4*len:len; + v = PyString_FromStringAndSize((char *)NULL, newlen); + if (v == NULL) + return NULL; + p = buf = PyString_AsString(v); + end = s + len; + while (s < end) { + if (*s != '\\') { + non_esc: + if (recode_encoding && (*s & 0x80)) { + PyObject *u, *w; + char *r; + const char* t; + Py_ssize_t rn; + t = s; + /* Decode non-ASCII bytes as UTF-8. */ + while (t < end && (*t & 0x80)) t++; + u = PyUnicode_DecodeUTF8(s, t - s, errors); + if(!u) goto failed; + + /* Recode them in target encoding. */ + w = PyUnicode_AsEncodedString( + u, recode_encoding, errors); + Py_DECREF(u); + if (!w) goto failed; + + /* Append bytes to output buffer. */ + assert(PyString_Check(w)); + r = PyString_AS_STRING(w); + rn = PyString_GET_SIZE(w); + Py_MEMCPY(p, r, rn); + p += rn; + Py_DECREF(w); + s = t; + } else { + *p++ = *s++; + } + continue; + } + s++; + if (s==end) { + PyErr_SetString(PyExc_ValueError, + "Trailing \\ in string"); + goto failed; + } + switch (*s++) { + /* XXX This assumes ASCII! */ + case '\n': break; + case '\\': *p++ = '\\'; break; + case '\'': *p++ = '\''; break; + case '\"': *p++ = '\"'; break; + case '/': *p++ = '/'; break; + case 'b': *p++ = '\b'; break; + case 'f': *p++ = '\014'; break; /* FF */ + case 't': *p++ = '\t'; break; + case 'n': *p++ = '\n'; break; + case 'r': *p++ = '\r'; break; + case 'v': *p++ = '\013'; break; /* VT */ + case 'a': *p++ = '\007'; break; /* BEL, not classic C */ + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + c = s[-1] - '0'; + if (s < end && '0' <= *s && *s <= '7') { + c = (c<<3) + *s++ - '0'; + if (s < end && '0' <= *s && *s <= '7') + c = (c<<3) + *s++ - '0'; + } + *p++ = c; + break; + case 'x': + if (s+1 < end && + isxdigit(Py_CHARMASK(s[0])) && + isxdigit(Py_CHARMASK(s[1]))) + { + unsigned int x = 0; + c = Py_CHARMASK(*s); + s++; + if (isdigit(c)) + x = c - '0'; + else if (islower(c)) + x = 10 + c - 'a'; + else + x = 10 + c - 'A'; + x = x << 4; + c = Py_CHARMASK(*s); + s++; + if (isdigit(c)) + x += c - '0'; + else if (islower(c)) + x += 10 + c - 'a'; + else + x += 10 + c - 'A'; + *p++ = x; + break; + } + if (!errors || strcmp(errors, "strict") == 0) { + PyErr_SetString(PyExc_ValueError, + "invalid \\x escape"); + goto failed; + } + if (strcmp(errors, "replace") == 0) { + *p++ = '?'; + } else if (strcmp(errors, "ignore") == 0) + /* do nothing */; + else { + PyErr_Format(PyExc_ValueError, + "decoding error; " + "unknown error handling code: %.400s", + errors); + goto failed; + } + default: + *p++ = '\\'; + s--; + goto non_esc; /* an arbitry number of unescaped + UTF-8 bytes may follow. */ + } + } + if (p-buf < newlen) + _PyString_Resize(&v, p - buf); + return v; + failed: + Py_DECREF(v); + return NULL; +} + +static +int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler, + const char *encoding, const char *reason, + const char *input, Py_ssize_t insize, Py_ssize_t *startinpos, Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, + PyObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr) +{ + static char *argparse = "O!n;decoding error handler must return (unicode, int) tuple"; + + PyObject *restuple = NULL; + PyObject *repunicode = NULL; + Py_ssize_t outsize = PyUnicode_GET_SIZE(*output); + Py_ssize_t requiredsize; + Py_ssize_t newpos; + Py_UNICODE *repptr; + Py_ssize_t repsize; + int res = -1; + + if (*errorHandler == NULL) { + *errorHandler = PyCodec_LookupError(errors); + if (*errorHandler == NULL) + goto onError; + } + + if (*exceptionObject == NULL) { + *exceptionObject = PyUnicodeDecodeError_Create( + encoding, input, insize, *startinpos, *endinpos, reason); + if (*exceptionObject == NULL) + goto onError; + } + else { + if (PyUnicodeDecodeError_SetStart(*exceptionObject, *startinpos)) + goto onError; + if (PyUnicodeDecodeError_SetEnd(*exceptionObject, *endinpos)) + goto onError; + if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason)) + goto onError; + } + + restuple = PyObject_CallFunctionObjArgs(*errorHandler, *exceptionObject, NULL); + if (restuple == NULL) + goto onError; + if (!PyTuple_Check(restuple)) { + PyErr_Format(PyExc_TypeError, &argparse[4]); + goto onError; + } + if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos)) + goto onError; + if (newpos<0) + newpos = insize+newpos; + if (newpos<0 || newpos>insize) { + PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos); + goto onError; + } + + /* need more space? (at least enough for what we + have+the replacement+the rest of the string (starting + at the new input position), so we won't have to check space + when there are no errors in the rest of the string) */ + repptr = PyUnicode_AS_UNICODE(repunicode); + repsize = PyUnicode_GET_SIZE(repunicode); + requiredsize = *outpos + repsize + insize-newpos; + if (requiredsize > outsize) { + if (requiredsize<2*outsize) + requiredsize = 2*outsize; + if (PyUnicode_Resize(output, requiredsize) < 0) + goto onError; + *outptr = PyUnicode_AS_UNICODE(*output) + *outpos; + } + *endinpos = newpos; + *inptr = input + newpos; + Py_UNICODE_COPY(*outptr, repptr, repsize); + *outptr += repsize; + *outpos += repsize; + /* we made it! */ + res = 0; + + onError: + Py_XDECREF(restuple); + return res; +} + +static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL; + +PyObject *_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) +{ + const char *starts = s; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + Py_ssize_t outpos; + int i; + PyObject *v; + Py_UNICODE *p; + const char *end; + char* message; + Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */ + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + + /* Escaped strings will always be longer than the resulting + Unicode string, so we start with size here and then reduce the + length after conversion to the true value. + (but if the error callback returns a long replacement string + we'll have to allocate more space) */ + v = PyUnicode_FromUnicode((const Py_UNICODE*)NULL, size); + if (v == NULL) + goto onError; + if (size == 0) + return (PyObject *)v; + + p = PyUnicode_AS_UNICODE(v); + end = s + size; + + while (s < end) { + unsigned char c; + Py_UNICODE x; + int digits; + + /* Non-escape characters are interpreted as Unicode ordinals */ + if (*s != '\\') { + *p++ = (unsigned char) *s++; + continue; + } + + startinpos = s-starts; + /* \ - Escapes */ + s++; + c = *s++; + if (s > end) + c = '\0'; /* Invalid after \ */ + switch (c) { + + /* \x escapes */ + case '\n': break; + case '\\': *p++ = '\\'; break; + case '\'': *p++ = '\''; break; + case '\"': *p++ = '\"'; break; + case '/': *p++ = '/'; break; + case 'b': *p++ = '\b'; break; + case 'f': *p++ = '\014'; break; /* FF */ + case 't': *p++ = '\t'; break; + case 'n': *p++ = '\n'; break; + case 'r': *p++ = '\r'; break; + case 'v': *p++ = '\013'; break; /* VT */ + case 'a': *p++ = '\007'; break; /* BEL, not classic C */ + + /* \OOO (octal) escapes */ + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + x = s[-1] - '0'; + if (s < end && '0' <= *s && *s <= '7') { + x = (x<<3) + *s++ - '0'; + if (s < end && '0' <= *s && *s <= '7') + x = (x<<3) + *s++ - '0'; + } + *p++ = x; + break; + + /* hex escapes */ + /* \xXX */ + case 'x': + digits = 2; + message = "truncated \\xXX escape"; + goto hexescape; + + /* \uXXXX */ + case 'u': + digits = 4; + message = "truncated \\uXXXX escape"; + goto hexescape; + + /* \UXXXXXXXX */ + case 'U': + digits = 8; + message = "truncated \\UXXXXXXXX escape"; + hexescape: + chr = 0; + outpos = p-PyUnicode_AS_UNICODE(v); + if (s+digits>end) { + endinpos = size; + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicodeescape", "end of string in escape sequence", + starts, size, &startinpos, &endinpos, &exc, &s, + (PyObject **)&v, &outpos, &p)) + goto onError; + goto nextByte; + } + for (i = 0; i < digits; ++i) { + c = (unsigned char) s[i]; + if (!isxdigit(c)) { + endinpos = (s+i+1)-starts; + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicodeescape", message, + starts, size, &startinpos, &endinpos, &exc, &s, + (PyObject **)&v, &outpos, &p)) + goto onError; + goto nextByte; + } + chr = (chr<<4) & ~0xF; + if (c >= '0' && c <= '9') + chr += c - '0'; + else if (c >= 'a' && c <= 'f') + chr += 10 + c - 'a'; + else + chr += 10 + c - 'A'; + } + s += i; + if (chr == 0xffffffff && PyErr_Occurred()) + /* _decoding_error will have already written into the + target buffer. */ + break; + store: + /* when we get here, chr is a 32-bit unicode character */ + if (chr <= 0xffff) + /* UCS-2 character */ + *p++ = (Py_UNICODE) chr; + else if (chr <= 0x10ffff) { + /* UCS-4 character. Either store directly, or as + surrogate pair. */ +#ifdef Py_UNICODE_WIDE + *p++ = chr; +#else + chr -= 0x10000L; + *p++ = 0xD800 + (Py_UNICODE) (chr >> 10); + *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF); +#endif + } else { + endinpos = s-starts; + outpos = p-PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicodeescape", "illegal Unicode character", + starts, size, &startinpos, &endinpos, &exc, &s, + (PyObject **)&v, &outpos, &p)) + goto onError; + } + break; + + /* \N{name} */ + case 'N': + message = "malformed \\N character escape"; + if (ucnhash_CAPI == NULL) { + /* load the unicode data module */ + PyObject *m, *api; + m = PyImport_ImportModule("unicodedata"); + if (m == NULL) + goto ucnhashError; + api = PyObject_GetAttrString(m, "ucnhash_CAPI"); + Py_DECREF(m); + if (api == NULL) + goto ucnhashError; + ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCObject_AsVoidPtr(api); + Py_DECREF(api); + if (ucnhash_CAPI == NULL) + goto ucnhashError; + } + if (*s == '{') { + const char *start = s+1; + /* look for the closing brace */ + while (*s != '}' && s < end) + s++; + if (s > start && s < end && *s == '}') { + /* found a name. look it up in the unicode database */ + message = "unknown Unicode character name"; + s++; +#if PY_VERSION_HEX < 0x02050000 + if (ucnhash_CAPI->getcode(start, (int)(s-start-1), &chr)) +#else + if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), &chr)) +#endif + goto store; + } + } + endinpos = s-starts; + outpos = p-PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicodeescape", message, + starts, size, &startinpos, &endinpos, &exc, &s, + (PyObject **)&v, &outpos, &p)) + goto onError; + break; + + default: + if (s > end) { + message = "\\ at end of string"; + s--; + endinpos = s-starts; + outpos = p-PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicodeescape", message, + starts, size, &startinpos, &endinpos, &exc, &s, + (PyObject **)&v, &outpos, &p)) + goto onError; + } + else { + *p++ = '\\'; + *p++ = (unsigned char)s[-1]; + } + break; + } + nextByte: + ; + } + if (PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + goto onError; + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return (PyObject *)v; + +ucnhashError: + PyErr_SetString( + PyExc_UnicodeError, + "\\N escapes not supported (can't load unicodedata module)" + ); + Py_XDECREF(v); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return NULL; + +onError: + Py_XDECREF(v); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return NULL; +} + static PyObject* decode_string(JSONData *jsondata) @@ -151,6 +623,7 @@ decode_string(JSONData *jsondata) case 't': case 'b': case 'f': + case '/': case '\\': string_escape = True; break; @@ -163,9 +636,9 @@ decode_string(JSONData *jsondata) len = ptr - jsondata->ptr - 1; if (has_unicode || jsondata->all_unicode) - object = PyUnicode_DecodeUnicodeEscape(jsondata->ptr+1, len, NULL); + object = _DecodeUnicodeEscape(jsondata->ptr+1, len, NULL); else if (string_escape) - object = PyString_DecodeEscape(jsondata->ptr+1, len, NULL, 0, NULL); + object = _DecodeEscape(jsondata->ptr+1, len, NULL, 0, NULL); else object = PyString_FromStringAndSize(jsondata->ptr+1, len); diff --git a/jsontest.py b/jsontest.py index 5885e97..7d95bd9 100755 --- a/jsontest.py +++ b/jsontest.py @@ -49,9 +49,10 @@ class JsonTest(unittest.TestCase): obj = cjson.decode(r'"\""') self.assertEqual(r'"', obj) -# def testReadEscapedSolidus(self): -# obj = cjson.decode(r'"\/"') -# self.assertEqual(r'/', obj) + def testReadEscapedSolidus(self): + obj = cjson.decode(r'"\/"') + print '%r == %r' % (r'/', obj) + self.assertEqual(r'/', obj) def testReadEscapedReverseSolidus(self): obj = cjson.decode(r'"\\"') @@ -141,7 +142,7 @@ class JsonTest(unittest.TestCase): def doReadBadArray(self): cjson.decode('[1,2,3,,]') - + def testReadBadObjectSyntax(self): self.assertRaises(_exception, self.doReadBadObjectSyntax) @@ -159,7 +160,7 @@ class JsonTest(unittest.TestCase): def testReadNegativeIntegerValue(self): obj = cjson.decode('{ "key" : -44 }') self.assertEqual({ "key" : -44 }, obj) - + def testReadFloatValue(self): obj = cjson.decode('{ "age" : 44.5 }') self.assertEqual({ "age" : 44.5 }, obj) @@ -176,7 +177,7 @@ class JsonTest(unittest.TestCase): def testReadSmallObject(self): obj = cjson.decode('{ "name" : "Patrick", "age":44} ') - self.assertEqual({ "age" : 44, "name" : "Patrick" }, obj) + self.assertEqual({ "age" : 44, "name" : "Patrick" }, obj) def testReadEmptyArray(self): obj = cjson.decode('[]') @@ -316,7 +317,7 @@ class JsonTest(unittest.TestCase): def testWriteLong(self): self.assertEqual("12345678901234567890", cjson.encode(12345678901234567890)) - + def main(): unittest.main() -- 1.5.6.5