Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
Cloud:OpenStack:Newton
python-ovs
0001-Add-optional-C-extension-wrapper-for-Pytho...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 0001-Add-optional-C-extension-wrapper-for-Python-JSON-par.patch of Package python-ovs
From c63b04d678aab6e63551f4105d6fc03c195d639e Mon Sep 17 00:00:00 2001 From: Terry Wilson <twilson@redhat.com> Date: Wed, 8 Jun 2016 08:55:14 -0500 Subject: [PATCH] Add optional C extension wrapper for Python JSON parsing The pure Python in-tree JSON parser is *much* slower than the in-tree C JSON parser. A local test parsing a 100Mb JSON file showed the Python version taking 270 seconds. With the C wrapper, it took under 4 seconds. The C extension will be used automatically if it can be built. If the extension fails to build, a warning is displayed and the build is restarted without the extension. The Serializer class is replaced with Python's built-in JSON library since the ability to process chunked data is not needed in that case. The extension should work with both Python 2.7 and Python 3.3+. Signed-off-by: Terry Wilson <twilson@redhat.com> Signed-off-by: Ben Pfaff <blp@ovn.org> --- Makefile.am | 2 +- python/automake.mk | 3 + python/ovs/_json.c | 268 +++++++++++++++++++++++++++++++++++++++++++++ python/ovs/json.py | 11 ++ python/setup.py | 51 ++++++++- 5 files changed, 332 insertions(+), 3 deletions(-) create mode 100644 python/ovs/_json.c Index: ovs-2.5.0/ovs/_json.c =================================================================== --- /dev/null +++ ovs-2.5.0/ovs/_json.c @@ -0,0 +1,268 @@ +#include "Python.h" +#include "lib/json.h" +#include "structmember.h" + +#if PY_MAJOR_VERSION >= 3 +#define IS_PY3K +#endif + +typedef struct { + PyObject_HEAD + struct json_parser *_parser; +} json_ParserObject; + +static void +Parser_dealloc(json_ParserObject * p) +{ + json_parser_abort(p->_parser); + Py_TYPE(p)->tp_free(p); +} + +static PyObject * +Parser_new(PyTypeObject * type, PyObject * args, PyObject * kwargs) +{ + json_ParserObject *self; + static char *kwlist[] = { "check_trailer", NULL }; + PyObject *check_trailer = NULL; + int ct_int = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, + &check_trailer)) { + return NULL; + } + + if (check_trailer != NULL) { + ct_int = PyObject_IsTrue(check_trailer); + if (ct_int < 0) { + return NULL; + } else if (ct_int) { + ct_int = JSPF_TRAILER; + } + } + + self = (json_ParserObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->_parser = json_parser_create(ct_int); + } + + return (PyObject *) self; +} + +static PyObject * +Parser_feed(json_ParserObject * self, PyObject * args) +{ + Py_ssize_t input_sz; + PyObject *input; + size_t rd; + char *input_str; + + if (self->_parser == NULL) { + return NULL; + } + + if (!PyArg_UnpackTuple(args, "input", 1, 1, &input)) { + return NULL; + } +#ifdef IS_PY3K + if ((input_str = PyUnicode_AsUTF8AndSize(input, &input_sz)) == NULL) { +#else + if (PyString_AsStringAndSize(input, &input_str, &input_sz) < 0) { +#endif + return NULL; + } + + rd = json_parser_feed(self->_parser, input_str, (size_t) input_sz); + +#ifdef IS_PY3K + return PyLong_FromSize_t(rd); +#else + return PyInt_FromSize_t(rd); +#endif +} + +static PyObject * +Parser_is_done(json_ParserObject * self) +{ + if (self->_parser == NULL) { + return NULL; + } + return PyBool_FromLong(json_parser_is_done(self->_parser)); +} + +static PyObject * +json_to_python(struct json *json) +{ + switch (json->type) { + case JSON_NULL: + Py_RETURN_NONE; + case JSON_FALSE: + Py_RETURN_FALSE; + case JSON_TRUE: + Py_RETURN_TRUE; + case JSON_OBJECT:{ + struct shash_node *node; + PyObject *dict = PyDict_New(); + + if (dict == NULL) { + return PyErr_NoMemory(); + } + SHASH_FOR_EACH(node, json->u.object) { + PyObject *key = PyUnicode_FromString(node->name); + PyObject *val = json_to_python(node->data); + + if (!(key && val) || PyDict_SetItem(dict, key, val)) { + Py_XDECREF(key); + Py_XDECREF(val); + Py_XDECREF(dict); + return NULL; + } + + Py_XDECREF(key); + Py_XDECREF(val); + } + return dict; + } + case JSON_ARRAY:{ + int i; + PyObject *arr = PyList_New(json->u.array.n); + + if (arr == NULL) { + return PyErr_NoMemory(); + } + for (i = 0; i < json->u.array.n; i++) { + PyObject *item = json_to_python(json->u.array.elems[i]); + + if (!item || PyList_SetItem(arr, i, item)) { + Py_XDECREF(arr); + return NULL; + } + } + return arr; + } + case JSON_REAL: + if (json->u.real != 0) { + return PyFloat_FromDouble(json->u.real); + } /* fall through to treat 0 as int */ + case JSON_INTEGER: +#ifdef IS_PY3K + return PyLong_FromLong((long) json->u.integer); +#else + return PyInt_FromLong((long) json->u.integer); +#endif + + case JSON_STRING: + return PyUnicode_FromString(json->u.string); + default: + return NULL; + } +} + +static PyObject * +Parser_finish(json_ParserObject * self) +{ + struct json *json; + PyObject *obj; + + if (self->_parser == NULL) { + return NULL; + } + + json = json_parser_finish(self->_parser); + self->_parser = NULL; + obj = json_to_python(json); + return obj; +} + +static PyMethodDef Parser_methods[] = { + {"feed", (PyCFunction) Parser_feed, METH_VARARGS, + "Feed data to the parser and return the index of the last object."}, + {"is_done", (PyCFunction) Parser_is_done, METH_NOARGS, + "Whether the parser has finished decoding an object."}, + {"finish", (PyCFunction) Parser_finish, METH_NOARGS, + "Finish parsing and return Python object parsed."}, + {NULL}, +}; + +static PyTypeObject json_ParserType = { + PyVarObject_HEAD_INIT(NULL, 0) + "ovs._json.Parser", /* tp_name */ + sizeof (json_ParserObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor) Parser_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + "Parser objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + Parser_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + Parser_new, /* tp_new */ +}; + +#ifdef IS_PY3K +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "ovs._json", /* m_name */ + "OVS JSON Parser module", /* m_doc */ + 0, /* m_size */ + 0, /* m_methods */ + 0, /* m_slots */ + 0, /* m_traverse */ + 0, /* m_clear */ + 0, /* m_free */ +}; + +#define INITERROR return NULL +#else /* !IS_PY3K */ +#define INITERROR return +#endif + +PyMODINIT_FUNC +#ifdef IS_PY3K +PyInit__json(void) +#else +init_json(void) +#endif +{ + PyObject *m; + + if (PyType_Ready(&json_ParserType) < 0) { + INITERROR; + } +#ifdef IS_PY3K + m = PyModule_Create(&moduledef); +#else + m = Py_InitModule3("ovs._json", NULL, "OVS JSON Parser module"); +#endif + + Py_INCREF(&json_ParserType); + PyModule_AddObject(m, "Parser", (PyObject *) & json_ParserType); +#ifdef IS_PY3K + return m; +#endif +} Index: ovs-2.5.0/ovs/json.py =================================================================== --- ovs-2.5.0.orig/ovs/json.py +++ ovs-2.5.0/ovs/json.py @@ -19,6 +19,11 @@ import re import StringIO import sys +try: + import ovs._json +except ImportError: + pass + __pychecker__ = 'no-stringiter' SPACES_PER_LEVEL = 2 @@ -76,6 +81,12 @@ class Parser(object): ## Maximum height of parsing stack. ## MAX_HEIGHT = 1000 + def __new__(cls, *args, **kwargs): + try: + return ovs._json.Parser(*args, **kwargs) + except NameError: + return super(Parser, cls).__new__(cls) + def __init__(self, check_trailer=False): self.check_trailer = check_trailer Index: ovs-2.5.0/setup.py =================================================================== --- ovs-2.5.0.orig/setup.py +++ ovs-2.5.0/setup.py @@ -13,6 +13,10 @@ from __future__ import print_function import sys +from distutils.command.build_ext import build_ext +from distutils.errors import CCompilerError, DistutilsExecError, \ + DistutilsPlatformError + import setuptools VERSION = "unknown" @@ -25,8 +29,33 @@ except IOError: file=sys.stderr) sys.exit(-1) +ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) +if sys.platform == 'win32': + ext_errors += (IOError, ValueError) + + +class BuildFailed(Exception): + pass + + +class try_build_ext(build_ext): + # This class allows C extension building to fail + # NOTE: build_ext is not a new-style class + + def run(self): + try: + build_ext.run(self) + except DistutilsPlatformError: + raise BuildFailed() -setuptools.setup( + def build_extension(self, ext): + try: + build_ext.build_extension(self, ext) + except ext_errors: + raise BuildFailed() + + +setup_args = dict( name='ovs', description='Open vSwitch library', version=VERSION, @@ -42,5 +71,23 @@ setuptools.setup( 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Networking', 'License :: OSI Approved :: Apache Software License' - ] + ], + ext_modules=[setuptools.Extension("ovs._json", sources=["ovs/_json.c"], + libraries=['openvswitch'])], + cmdclass={'build_ext': try_build_ext}, ) + +try: + setuptools.setup(**setup_args) +except BuildFailed: + BUILD_EXT_WARNING = ("WARNING: The C extension could not be compiled, " + "speedups are not enabled.") + print("*" * 75) + print(BUILD_EXT_WARNING) + print("Failure information, if any, is above.") + print("Retrying the build without the C extension.") + print("*" * 75) + + del(setup_args['cmdclass']) + del(setup_args['ext_modules']) + setuptools.setup(**setup_args)
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
