From 898fbe0d33af1be6d917f96ac374d3867c5ed5cb Mon Sep 17 00:00:00 2001 From: "R. Church" Date: Sun, 17 Feb 2013 00:04:25 -0800 Subject: [PATCH 1/3] this works --- flojay/decoder_callbacks.c | 239 ++++++++++++++++++++++++++++++++ flojay/flojay.c | 4 + setup.py | 3 +- tests/decoder_callback_tests.py | 52 +++++++ 4 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 flojay/decoder_callbacks.c create mode 100644 tests/decoder_callback_tests.py diff --git a/flojay/decoder_callbacks.c b/flojay/decoder_callbacks.c new file mode 100644 index 0000000..a2a2fbb --- /dev/null +++ b/flojay/decoder_callbacks.c @@ -0,0 +1,239 @@ +#include + +typedef struct { + PyObject_HEAD + PyObject * root; + PyObject * containers; + PyObject * map_key; +} flojay_PythonDecoderCallbacks_object; + +static inline flojay_PythonDecoderCallbacks_object * +_self(PyObject * pyself) +{ + return (flojay_PythonDecoderCallbacks_object *) pyself; +} + +static void +append_value(flojay_PythonDecoderCallbacks_object * self, + PyObject * value) +{ + PyObject * current_container; + if(Py_None == self->root) { + self->root = value; + } else if(NULL != self->map_key) { + current_container = PyList_GetItem(self->containers, 0); + PyDict_SetItem(current_container, self->map_key, value); + self->map_key = NULL; + } else { + current_container = PyList_GetItem(self->containers, 0); + PyList_Append(current_container, value); + } +} + +static int +flojay_PythonDecoderCallbacks_init(PyObject * pyself) +{ + flojay_PythonDecoderCallbacks_object * self = _self(pyself); + self->root = Py_None; + self->map_key = Py_None; + self->containers = PyList_New(0); + return 0; +} + +static PyObject * +flojay_PythonDecoderCallbacks_root(PyObject * pyself) +{ + return _self(pyself)->root; +} + +static PyObject * +flojay_PythonDecoderCallbacks_handle_value(PyObject * pyself, PyObject * args) +{ + PyObject * value; + if(!PyArg_ParseTuple(args, "O", &value)) + return 0; + append_value(_self(pyself), value); + return Py_None; +} + +static PyObject * +flojay_PythonDecoderCallbacks_handle_null(PyObject * pyself, PyObject * args) +{ + static PyObject * none_tuple = NULL; + if(NULL == none_tuple) { + none_tuple = Py_BuildValue("(O)", Py_None); + Py_INCREF(none_tuple); + } + + return flojay_PythonDecoderCallbacks_handle_value(pyself, none_tuple); +} + +static void +start_container(flojay_PythonDecoderCallbacks_object * self, + PyObject * container) +{ + append_value(self, container); + PyList_Insert(self->containers, 0, container); +} + +static void +pop_container(flojay_PythonDecoderCallbacks_object * self) +{ + PyList_SetSlice(self->containers, 0, 1, NULL); +} + +static PyObject * +flojay_PythonDecoderCallbacks_start_array(PyObject * pyself) +{ + PyObject * container = PyList_New(0); + start_container(_self(pyself), container); + return Py_None; +} + +static PyObject * +flojay_PythonDecoderCallbacks_end_array(PyObject * pyself) +{ + pop_container(_self(pyself)); + return Py_None; +} + +static PyObject * +flojay_PythonDecoderCallbacks_start_map(PyObject * pyself) +{ + PyObject * container = PyDict_New(); + start_container(_self(pyself), container); + return Py_None; +} + +static PyObject * +flojay_PythonDecoderCallbacks_map_key(PyObject * pyself, PyObject * args) +{ + PyObject * map_key; + if(!PyArg_ParseTuple(args, "O", &map_key)) + return 0; + + _self(pyself)->map_key = map_key; + + return Py_None; +} + +static PyObject * +flojay_PythonDecoderCallbacks_end_map(PyObject * pyself) +{ + pop_container(_self(pyself)); + return Py_None; +} + +static PyMethodDef flojay_PythonDecoderCallbacks_methods[] = { + { + "__init__", + (PyCFunction)flojay_PythonDecoderCallbacks_init, 1, + "Init!" + }, + { + "root", + (PyCFunction)flojay_PythonDecoderCallbacks_root, 1, + "Return the root object of the Python object graph produced by the " + "parser." + }, + { + "handle_number", + (PyCFunction)flojay_PythonDecoderCallbacks_handle_value, 2, + NULL + }, + { + "handle_string", + (PyCFunction)flojay_PythonDecoderCallbacks_handle_value, 2, + NULL + }, + { + "handle_boolean", + (PyCFunction)flojay_PythonDecoderCallbacks_handle_value, 2, + NULL + }, + { + "handle_null", + (PyCFunction)flojay_PythonDecoderCallbacks_handle_null, 1, + NULL + }, + { + "handle_start_array", + (PyCFunction)flojay_PythonDecoderCallbacks_start_array, 1, + NULL + }, + { + "handle_end_array", + (PyCFunction)flojay_PythonDecoderCallbacks_end_array, 1, + NULL + }, + { + "handle_start_map", + (PyCFunction)flojay_PythonDecoderCallbacks_start_map, 1, + NULL + }, + { + "handle_map_key", + (PyCFunction)flojay_PythonDecoderCallbacks_map_key, 2, + NULL + }, + { + "handle_end_map", + (PyCFunction)flojay_PythonDecoderCallbacks_end_map, 1, + NULL + }, + {NULL} /* Sentinel */ +}; + +static PyTypeObject flojay_PythonDecoderCallbacks_type = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "flojay.PythonDecoderCallbacks", /*tp_name*/ + sizeof(flojay_PythonDecoderCallbacks_object), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "flojay callbacks for decoding JSON to Python", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + flojay_PythonDecoderCallbacks_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)flojay_PythonDecoderCallbacks_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew /* tp_new */ +}; + +void +init_flojay_decoder_callbacks(PyObject * module) +{ + if (PyType_Ready(&flojay_PythonDecoderCallbacks_type) < 0) + return; + + Py_INCREF(&flojay_PythonDecoderCallbacks_type); + + PyModule_AddObject(module, "PythonDecoderCallbacks", + (PyObject *)&flojay_PythonDecoderCallbacks_type); +} diff --git a/flojay/flojay.c b/flojay/flojay.c index ad661c0..c92fa0d 100644 --- a/flojay/flojay.c +++ b/flojay/flojay.c @@ -52,6 +52,8 @@ flojay_generator_new(PyTypeObject * type, PyObject * args, PyObject * kwds) return (PyObject *) self; } +void init_flojay_decoder_callbacks(PyObject * module); + static int _flojay_handle_yajl_error(yajl_gen_status status) { @@ -776,6 +778,8 @@ initflojay(void) PyModule_AddObject(m, "JSONEventParser", (PyObject *)&flojay_JSONEventParserType); + init_flojay_decoder_callbacks(m); + allocate_method_names(); } diff --git a/setup.py b/setup.py index 6b3a7ad..7ae149b 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,8 @@ ('MINOR_VERSION', '1')], extra_compile_args=['--std=c99'], include_dirs=['flojay/lloyd-yajl/src'], - sources=yajl_sources + ['flojay/flojay.c']) + sources=yajl_sources + ['flojay/flojay.c', + 'flojay/decoder_callbacks.c']) setup(name='flojay', diff --git a/tests/decoder_callback_tests.py b/tests/decoder_callback_tests.py new file mode 100644 index 0000000..644374f --- /dev/null +++ b/tests/decoder_callback_tests.py @@ -0,0 +1,52 @@ +import flojay +from unittest import TestCase +from nose.tools import eq_, raises + + +class DecoderCallbackTests(TestCase): + def loads(self, json_string): + callbacks = flojay.PythonDecoderCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse(json_string) + return callbacks.root + + def test_array_of_numbers(self): + eq_(self.loads("[1, 2, 3]"), + [1, 2, 3]) + + def test_nested_array_of_numbers(self): + eq_(self.loads("[1, [2, [4, 5], 6]]"), + [1, [2, [4, 5], 6]]) + + def test_strings_booleans_and_none(self): + eq_(self.loads(u'["Ren\xc3e", false, true, null, null]'), + [u"Ren\xc3e", False, True, None, None]) + + def test_simple_map(self): + eq_(self.loads('{"a": 1, "b": 2, "c": 3}'), + {"a": 1, "b": 2, "c": 3}) + + def test_nested_map(self): + eq_(self.loads('{"a": {"a": 1}, "b": {"bb": {"c": {"cc": 3}}, "d": 4}}'), + {"a": {"a": 1}, "b": {"bb": {"c": {"cc": 3}}, "d": 4}}) + + def test_arrays_and_maps_together(self): + eq_(self.loads('[{"a": [1, 2, {"b": 3}]}, "c", [{"d": 4}, "e", [{}], []]'), + [{"a": [1, 2, {"b": 3}]}, "c", [{"d": 4}, "e", [{}], []]]) + + def test_root(self): + callbacks = flojay.PythonDecoderCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse('["a","b') + eq_(callbacks.root, ["a"]) + p.parse('", {"c":') + eq_(callbacks.root, ["a", "b", {}]) + p.parse('1, "d": 2,') + eq_(callbacks.root, ["a", "b", {"c": 1, "d": 2}]) + p.parse('"e": 3}, "f"]') + eq_(callbacks.root, ["a", "b", {"c": 1, "d": 2, "e": 3}, "f"]) + + + + + From c7ce83afa0bc295369e1997952f7d037df562a01 Mon Sep 17 00:00:00 2001 From: "R. Church" Date: Sun, 17 Feb 2013 01:26:42 -0800 Subject: [PATCH 2/3] * Added reference counting management * Changed root to an attribute. I'll probably wind up changing it back. I'm still working out how to make it so that the user can consume top-level objects as they're parsed. --- flojay/decoder_callbacks.c | 41 ++++++++++++++++++++++++--------- tests/decoder_callback_tests.py | 1 - 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/flojay/decoder_callbacks.c b/flojay/decoder_callbacks.c index a2a2fbb..56f55ec 100644 --- a/flojay/decoder_callbacks.c +++ b/flojay/decoder_callbacks.c @@ -18,12 +18,15 @@ append_value(flojay_PythonDecoderCallbacks_object * self, PyObject * value) { PyObject * current_container; + if(Py_None == self->root) { + Py_INCREF(value); self->root = value; - } else if(NULL != self->map_key) { + } else if(Py_None != self->map_key) { current_container = PyList_GetItem(self->containers, 0); PyDict_SetItem(current_container, self->map_key, value); - self->map_key = NULL; + Py_DECREF(self->map_key); + self->map_key = Py_None; } else { current_container = PyList_GetItem(self->containers, 0); PyList_Append(current_container, value); @@ -37,11 +40,21 @@ flojay_PythonDecoderCallbacks_init(PyObject * pyself) self->root = Py_None; self->map_key = Py_None; self->containers = PyList_New(0); + Py_INCREF(self->containers); return 0; } +static void +flojay_PythonDecoderCallbacks_dealloc(PyObject * pyself) +{ + flojay_PythonDecoderCallbacks_object * self = _self(pyself); + Py_XDECREF(self->root); + Py_XDECREF(self->map_key); + Py_XDECREF(self->containers); +} + static PyObject * -flojay_PythonDecoderCallbacks_root(PyObject * pyself) +flojay_PythonDecoderCallbacks_get_root(PyObject * pyself) { return _self(pyself)->root; } @@ -112,6 +125,7 @@ flojay_PythonDecoderCallbacks_map_key(PyObject * pyself, PyObject * args) if(!PyArg_ParseTuple(args, "O", &map_key)) return 0; + Py_INCREF(map_key); _self(pyself)->map_key = map_key; return Py_None; @@ -130,12 +144,6 @@ static PyMethodDef flojay_PythonDecoderCallbacks_methods[] = { (PyCFunction)flojay_PythonDecoderCallbacks_init, 1, "Init!" }, - { - "root", - (PyCFunction)flojay_PythonDecoderCallbacks_root, 1, - "Return the root object of the Python object graph produced by the " - "parser." - }, { "handle_number", (PyCFunction)flojay_PythonDecoderCallbacks_handle_value, 2, @@ -184,13 +192,24 @@ static PyMethodDef flojay_PythonDecoderCallbacks_methods[] = { {NULL} /* Sentinel */ }; +static PyGetSetDef flojay_PythonDecoderCallbacks_getset[] = { + {"root", + (getter)flojay_PythonDecoderCallbacks_get_root, + (setter)NULL, + "The root object of the Python object graph produced by the " + "parser.", + NULL}, + {NULL} /* Sentinel */ +}; + + static PyTypeObject flojay_PythonDecoderCallbacks_type = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "flojay.PythonDecoderCallbacks", /*tp_name*/ sizeof(flojay_PythonDecoderCallbacks_object), /*tp_basicsize*/ 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ + (destructor)flojay_PythonDecoderCallbacks_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -215,7 +234,7 @@ static PyTypeObject flojay_PythonDecoderCallbacks_type = { 0, /* tp_iternext */ flojay_PythonDecoderCallbacks_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + flojay_PythonDecoderCallbacks_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ diff --git a/tests/decoder_callback_tests.py b/tests/decoder_callback_tests.py index 644374f..b239685 100644 --- a/tests/decoder_callback_tests.py +++ b/tests/decoder_callback_tests.py @@ -47,6 +47,5 @@ def test_root(self): eq_(callbacks.root, ["a", "b", {"c": 1, "d": 2, "e": 3}, "f"]) - From 983c91fdcc99a73a29c468462252c85aa398b23a Mon Sep 17 00:00:00 2001 From: "R. Church" Date: Sun, 24 Feb 2013 17:53:37 -0800 Subject: [PATCH 3/3] Changes to the decoder callbacks so that it's possible to take completed results as parsing continues. You can take the contents of root and then allow them to be garbage collected by deleting the list elements. --- flojay/decoder_callbacks.c | 42 ++++++++++++++++----------------- tests/decoder_callback_tests.py | 29 ++++++++++++++++------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/flojay/decoder_callbacks.c b/flojay/decoder_callbacks.c index 56f55ec..abcd4ae 100644 --- a/flojay/decoder_callbacks.c +++ b/flojay/decoder_callbacks.c @@ -2,9 +2,8 @@ typedef struct { PyObject_HEAD - PyObject * root; PyObject * containers; - PyObject * map_key; + PyObject * map_keys; } flojay_PythonDecoderCallbacks_object; static inline flojay_PythonDecoderCallbacks_object * @@ -19,17 +18,15 @@ append_value(flojay_PythonDecoderCallbacks_object * self, { PyObject * current_container; - if(Py_None == self->root) { - Py_INCREF(value); - self->root = value; - } else if(Py_None != self->map_key) { - current_container = PyList_GetItem(self->containers, 0); - PyDict_SetItem(current_container, self->map_key, value); - Py_DECREF(self->map_key); - self->map_key = Py_None; - } else { + if(0 != PyList_Size(self->containers)) { current_container = PyList_GetItem(self->containers, 0); - PyList_Append(current_container, value); + if(PyDict_Check(current_container)) { + PyObject * map_key = PyList_GetItem(self->map_keys, 0); + PyDict_SetItem(current_container, map_key, value); + PyList_SetSlice(self->map_keys, 0, 1, NULL); + } else { + PyList_Append(current_container, value); + } } } @@ -37,8 +34,8 @@ static int flojay_PythonDecoderCallbacks_init(PyObject * pyself) { flojay_PythonDecoderCallbacks_object * self = _self(pyself); - self->root = Py_None; - self->map_key = Py_None; + self->map_keys = PyList_New(0); + Py_INCREF(self->map_keys); self->containers = PyList_New(0); Py_INCREF(self->containers); return 0; @@ -48,15 +45,15 @@ static void flojay_PythonDecoderCallbacks_dealloc(PyObject * pyself) { flojay_PythonDecoderCallbacks_object * self = _self(pyself); - Py_XDECREF(self->root); - Py_XDECREF(self->map_key); + Py_XDECREF(self->map_keys); Py_XDECREF(self->containers); } static PyObject * flojay_PythonDecoderCallbacks_get_root(PyObject * pyself) { - return _self(pyself)->root; + ssize_t len = PyList_Size(_self(pyself)->containers); + return PyList_GetItem(_self(pyself)->containers, len - 1); } static PyObject * @@ -85,14 +82,17 @@ static void start_container(flojay_PythonDecoderCallbacks_object * self, PyObject * container) { - append_value(self, container); PyList_Insert(self->containers, 0, container); } static void pop_container(flojay_PythonDecoderCallbacks_object * self) { - PyList_SetSlice(self->containers, 0, 1, NULL); + if(PyList_Size(self->containers) > 1) { + PyObject * container = PyList_GetItem(self->containers, 0); + PyList_SetSlice(self->containers, 0, 1, NULL); + append_value(self, container); + } } static PyObject * @@ -125,9 +125,7 @@ flojay_PythonDecoderCallbacks_map_key(PyObject * pyself, PyObject * args) if(!PyArg_ParseTuple(args, "O", &map_key)) return 0; - Py_INCREF(map_key); - _self(pyself)->map_key = map_key; - + PyList_Insert(_self(pyself)->map_keys, 0, map_key); return Py_None; } diff --git a/tests/decoder_callback_tests.py b/tests/decoder_callback_tests.py index b239685..cd054e4 100644 --- a/tests/decoder_callback_tests.py +++ b/tests/decoder_callback_tests.py @@ -8,14 +8,17 @@ def loads(self, json_string): callbacks = flojay.PythonDecoderCallbacks() p = flojay.JSONEventParser(callbacks) p.parse(json_string) - return callbacks.root + root = callbacks.root + print "root is ", root + return root def test_array_of_numbers(self): eq_(self.loads("[1, 2, 3]"), [1, 2, 3]) def test_nested_array_of_numbers(self): - eq_(self.loads("[1, [2, [4, 5], 6]]"), + root = self.loads("[1, [2, [4, 5], 6]]") + eq_(root, [1, [2, [4, 5], 6]]) def test_strings_booleans_and_none(self): @@ -31,21 +34,29 @@ def test_nested_map(self): {"a": {"a": 1}, "b": {"bb": {"c": {"cc": 3}}, "d": 4}}) def test_arrays_and_maps_together(self): - eq_(self.loads('[{"a": [1, 2, {"b": 3}]}, "c", [{"d": 4}, "e", [{}], []]'), + eq_(self.loads('[{"a": [1, 2, {"b": 3}]}, "c", [{"d": 4}, "e", [{}], []]]'), [{"a": [1, 2, {"b": 3}]}, "c", [{"d": 4}, "e", [{}], []]]) - def test_root(self): + def test_root_with_partial_results(self): callbacks = flojay.PythonDecoderCallbacks() p = flojay.JSONEventParser(callbacks) p.parse('["a","b') eq_(callbacks.root, ["a"]) p.parse('", {"c":') - eq_(callbacks.root, ["a", "b", {}]) + eq_(callbacks.root, ["a", "b"]) p.parse('1, "d": 2,') - eq_(callbacks.root, ["a", "b", {"c": 1, "d": 2}]) + eq_(callbacks.root, ["a", "b"]) p.parse('"e": 3}, "f"]') eq_(callbacks.root, ["a", "b", {"c": 1, "d": 2, "e": 3}, "f"]) - - - + def test_root_can_be_empited_out_now_and_again_with_no_apparent_ill_effects(self): + callbacks = flojay.PythonDecoderCallbacks() + p = flojay.JSONEventParser(callbacks) + p.parse('["a", "b"') + eq_(callbacks.root, ["a", "b"]) + del callbacks.root[0:1] + eq_(callbacks.root, []) + p.parse(', "c", "d"') + eq_(callbacks.root, ["c", "d"]) + p.parse("]") + eq_(callbacks.root, ["c", "d"])