mirror of https://git.kore.io/kore.git
Add a locking mechanism in pykore.
Support the async with syntax: lock = kore.lock() async with lock: # your block Fix some small issues with other parts of the python system.
This commit is contained in:
parent
1ac131c48f
commit
b70d1ee80f
|
@ -18,14 +18,17 @@
|
||||||
#define CORO_STATE_SUSPENDED 2
|
#define CORO_STATE_SUSPENDED 2
|
||||||
|
|
||||||
struct python_coro {
|
struct python_coro {
|
||||||
|
u_int64_t id;
|
||||||
int state;
|
int state;
|
||||||
int error;
|
int error;
|
||||||
PyObject *obj;
|
PyObject *obj;
|
||||||
|
struct pylock *lock;
|
||||||
struct http_request *request;
|
struct http_request *request;
|
||||||
TAILQ_ENTRY(python_coro) list;
|
TAILQ_ENTRY(python_coro) list;
|
||||||
};
|
};
|
||||||
|
|
||||||
static PyObject *python_kore_log(PyObject *, PyObject *);
|
static PyObject *python_kore_log(PyObject *, PyObject *);
|
||||||
|
static PyObject *python_kore_lock(PyObject *, PyObject *);
|
||||||
static PyObject *python_kore_bind(PyObject *, PyObject *);
|
static PyObject *python_kore_bind(PyObject *, PyObject *);
|
||||||
static PyObject *python_kore_fatal(PyObject *, PyObject *);
|
static PyObject *python_kore_fatal(PyObject *, PyObject *);
|
||||||
static PyObject *python_kore_queue(PyObject *, PyObject *);
|
static PyObject *python_kore_queue(PyObject *, PyObject *);
|
||||||
|
@ -47,6 +50,7 @@ static PyObject *python_websocket_broadcast(PyObject *, PyObject *);
|
||||||
|
|
||||||
static struct PyMethodDef pykore_methods[] = {
|
static struct PyMethodDef pykore_methods[] = {
|
||||||
METHOD("log", python_kore_log, METH_VARARGS),
|
METHOD("log", python_kore_log, METH_VARARGS),
|
||||||
|
METHOD("lock", python_kore_lock, METH_NOARGS),
|
||||||
METHOD("bind", python_kore_bind, METH_VARARGS),
|
METHOD("bind", python_kore_bind, METH_VARARGS),
|
||||||
METHOD("queue", python_kore_queue, METH_VARARGS),
|
METHOD("queue", python_kore_queue, METH_VARARGS),
|
||||||
METHOD("fatal", python_kore_fatal, METH_VARARGS),
|
METHOD("fatal", python_kore_fatal, METH_VARARGS),
|
||||||
|
@ -217,6 +221,64 @@ static PyTypeObject pyqueue_op_type = {
|
||||||
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct pylock {
|
||||||
|
PyObject_HEAD
|
||||||
|
struct python_coro *owner;
|
||||||
|
TAILQ_HEAD(, pylock_op) ops;
|
||||||
|
};
|
||||||
|
|
||||||
|
static PyObject *pylock_aexit(struct pylock *, PyObject *);
|
||||||
|
static PyObject *pylock_aenter(struct pylock *, PyObject *);
|
||||||
|
|
||||||
|
static PyMethodDef pylock_methods[] = {
|
||||||
|
METHOD("__aexit__", pylock_aexit, METH_VARARGS),
|
||||||
|
METHOD("__aenter__", pylock_aenter, METH_NOARGS),
|
||||||
|
METHOD(NULL, NULL, -1)
|
||||||
|
};
|
||||||
|
|
||||||
|
static void pylock_dealloc(struct pylock *);
|
||||||
|
|
||||||
|
static PyTypeObject pylock_type = {
|
||||||
|
PyVarObject_HEAD_INIT(NULL, 0)
|
||||||
|
.tp_name = "kore.lock",
|
||||||
|
.tp_doc = "locking mechanism",
|
||||||
|
.tp_methods = pylock_methods,
|
||||||
|
.tp_basicsize = sizeof(struct pylock),
|
||||||
|
.tp_dealloc = (destructor)pylock_dealloc,
|
||||||
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
|
||||||
|
};
|
||||||
|
|
||||||
|
struct pylock_op {
|
||||||
|
PyObject_HEAD
|
||||||
|
int locking;
|
||||||
|
int active;
|
||||||
|
struct pylock *lock;
|
||||||
|
struct python_coro *coro;
|
||||||
|
TAILQ_ENTRY(pylock_op) list;
|
||||||
|
};
|
||||||
|
|
||||||
|
static void pylock_op_dealloc(struct pylock_op *);
|
||||||
|
|
||||||
|
static PyObject *pylock_op_await(PyObject *);
|
||||||
|
static PyObject *pylock_op_iternext(struct pylock_op *);
|
||||||
|
|
||||||
|
static PyAsyncMethods pylock_op_async = {
|
||||||
|
(unaryfunc)pylock_op_await,
|
||||||
|
NULL,
|
||||||
|
NULL
|
||||||
|
};
|
||||||
|
|
||||||
|
static PyTypeObject pylock_op_type = {
|
||||||
|
PyVarObject_HEAD_INIT(NULL, 0)
|
||||||
|
.tp_name = "kore.lockop",
|
||||||
|
.tp_doc = "lock awaitable",
|
||||||
|
.tp_as_async = &pylock_op_async,
|
||||||
|
.tp_iternext = (iternextfunc)pylock_op_iternext,
|
||||||
|
.tp_basicsize = sizeof(struct pylock_op),
|
||||||
|
.tp_dealloc = (destructor)pylock_op_dealloc,
|
||||||
|
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
|
||||||
|
};
|
||||||
|
|
||||||
struct pyconnection {
|
struct pyconnection {
|
||||||
PyObject_HEAD
|
PyObject_HEAD
|
||||||
struct connection *c;
|
struct connection *c;
|
||||||
|
|
192
src/python.c
192
src/python.c
|
@ -38,7 +38,8 @@ static PyObject *python_callable(PyObject *, const char *);
|
||||||
static PyObject *pyhttp_file_alloc(struct http_file *);
|
static PyObject *pyhttp_file_alloc(struct http_file *);
|
||||||
static PyObject *pyhttp_request_alloc(const struct http_request *);
|
static PyObject *pyhttp_request_alloc(const struct http_request *);
|
||||||
|
|
||||||
static struct python_coro *python_coro_create(PyObject *);
|
static struct python_coro *python_coro_create(PyObject *,
|
||||||
|
struct http_request *);
|
||||||
static int python_coro_run(struct python_coro *);
|
static int python_coro_run(struct python_coro *);
|
||||||
static void python_coro_wakeup(struct python_coro *);
|
static void python_coro_wakeup(struct python_coro *);
|
||||||
|
|
||||||
|
@ -51,6 +52,8 @@ static PyObject *pysocket_async_send(struct pysocket_op *);
|
||||||
static PyObject *pysocket_async_accept(struct pysocket_op *);
|
static PyObject *pysocket_async_accept(struct pysocket_op *);
|
||||||
static PyObject *pysocket_async_connect(struct pysocket_op *);
|
static PyObject *pysocket_async_connect(struct pysocket_op *);
|
||||||
|
|
||||||
|
static void pylock_do_release(struct pylock *);
|
||||||
|
|
||||||
#if defined(KORE_USE_PGSQL)
|
#if defined(KORE_USE_PGSQL)
|
||||||
static PyObject *pykore_pgsql_alloc(struct http_request *,
|
static PyObject *pykore_pgsql_alloc(struct http_request *,
|
||||||
const char *, const char *);
|
const char *, const char *);
|
||||||
|
@ -142,6 +145,7 @@ static struct kore_pool coro_pool;
|
||||||
static struct kore_pool queue_wait_pool;
|
static struct kore_pool queue_wait_pool;
|
||||||
static struct kore_pool queue_object_pool;
|
static struct kore_pool queue_object_pool;
|
||||||
|
|
||||||
|
static u_int64_t coro_id;
|
||||||
static int coro_count;
|
static int coro_count;
|
||||||
static TAILQ_HEAD(, python_coro) coro_runnable;
|
static TAILQ_HEAD(, python_coro) coro_runnable;
|
||||||
static TAILQ_HEAD(, python_coro) coro_suspended;
|
static TAILQ_HEAD(, python_coro) coro_suspended;
|
||||||
|
@ -155,7 +159,9 @@ static struct python_coro *coro_running = NULL;
|
||||||
void
|
void
|
||||||
kore_python_init(void)
|
kore_python_init(void)
|
||||||
{
|
{
|
||||||
|
coro_id = 0;
|
||||||
coro_count = 0;
|
coro_count = 0;
|
||||||
|
|
||||||
TAILQ_INIT(&coro_runnable);
|
TAILQ_INIT(&coro_runnable);
|
||||||
TAILQ_INIT(&coro_suspended);
|
TAILQ_INIT(&coro_suspended);
|
||||||
|
|
||||||
|
@ -220,7 +226,10 @@ kore_python_coro_delete(void *obj)
|
||||||
|
|
||||||
coro = obj;
|
coro = obj;
|
||||||
coro_count--;
|
coro_count--;
|
||||||
|
|
||||||
|
coro_running = coro;
|
||||||
Py_DECREF(coro->obj);
|
Py_DECREF(coro->obj);
|
||||||
|
coro_running = NULL;
|
||||||
|
|
||||||
if (coro->state == CORO_STATE_RUNNABLE)
|
if (coro->state == CORO_STATE_RUNNABLE)
|
||||||
TAILQ_REMOVE(&coro_runnable, coro, list);
|
TAILQ_REMOVE(&coro_runnable, coro, list);
|
||||||
|
@ -241,21 +250,20 @@ kore_python_log_error(const char *function)
|
||||||
|
|
||||||
PyErr_Fetch(&type, &value, &traceback);
|
PyErr_Fetch(&type, &value, &traceback);
|
||||||
|
|
||||||
if (type == NULL || value == NULL || traceback == NULL) {
|
if (type == NULL || value == NULL) {
|
||||||
kore_log(LOG_ERR, "unknown python exception in '%s'", function);
|
kore_log(LOG_ERR, "unknown python exception in '%s'", function);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
kore_log(LOG_ERR,
|
kore_log(LOG_ERR,
|
||||||
"python exception in '%s' - type:%s - value:%s - trace:%s",
|
"python exception in '%s' - type:%s - value:%s",
|
||||||
function,
|
function,
|
||||||
PyUnicode_AsUTF8AndSize(type, NULL),
|
PyUnicode_AsUTF8AndSize(type, NULL),
|
||||||
PyUnicode_AsUTF8AndSize(value, NULL),
|
PyUnicode_AsUTF8AndSize(value, NULL));
|
||||||
PyUnicode_AsUTF8AndSize(traceback, NULL));
|
|
||||||
|
|
||||||
Py_DECREF(type);
|
Py_DECREF(type);
|
||||||
Py_DECREF(value);
|
Py_DECREF(value);
|
||||||
Py_DECREF(traceback);
|
Py_XDECREF(traceback);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void *
|
static void *
|
||||||
|
@ -320,7 +328,7 @@ python_module_getsym(struct kore_module *module, const char *symbol)
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct python_coro *
|
static struct python_coro *
|
||||||
python_coro_create(PyObject *obj)
|
python_coro_create(PyObject *obj, struct http_request *req)
|
||||||
{
|
{
|
||||||
struct python_coro *coro;
|
struct python_coro *coro;
|
||||||
|
|
||||||
|
@ -332,7 +340,8 @@ python_coro_create(PyObject *obj)
|
||||||
|
|
||||||
coro->obj = obj;
|
coro->obj = obj;
|
||||||
coro->error = 0;
|
coro->error = 0;
|
||||||
coro->request = req_running;
|
coro->request = req;
|
||||||
|
coro->id = coro_id++;
|
||||||
coro->state = CORO_STATE_RUNNABLE;
|
coro->state = CORO_STATE_RUNNABLE;
|
||||||
|
|
||||||
TAILQ_INSERT_HEAD(&coro_runnable, coro, list);
|
TAILQ_INSERT_HEAD(&coro_runnable, coro, list);
|
||||||
|
@ -458,7 +467,7 @@ python_runtime_http_request(void *addr, struct http_request *req)
|
||||||
|
|
||||||
if (PyCoro_CheckExact(pyret)) {
|
if (PyCoro_CheckExact(pyret)) {
|
||||||
http_request_sleep(req);
|
http_request_sleep(req);
|
||||||
req->py_coro = python_coro_create(pyret);
|
req->py_coro = python_coro_create(pyret, req);
|
||||||
req_running = NULL;
|
req_running = NULL;
|
||||||
/* XXX merge with the above python_coro_run() block. */
|
/* XXX merge with the above python_coro_run() block. */
|
||||||
if (python_coro_run(req->py_coro) == KORE_RESULT_OK) {
|
if (python_coro_run(req->py_coro) == KORE_RESULT_OK) {
|
||||||
|
@ -708,6 +717,7 @@ python_module_init(void)
|
||||||
if ((pykore = PyModule_Create(&pykore_module)) == NULL)
|
if ((pykore = PyModule_Create(&pykore_module)) == NULL)
|
||||||
fatal("python_module_init: failed to setup pykore module");
|
fatal("python_module_init: failed to setup pykore module");
|
||||||
|
|
||||||
|
python_push_type("pylock", pykore, &pylock_type);
|
||||||
python_push_type("pyqueue", pykore, &pyqueue_type);
|
python_push_type("pyqueue", pykore, &pyqueue_type);
|
||||||
python_push_type("pysocket", pykore, &pysocket_type);
|
python_push_type("pysocket", pykore, &pysocket_type);
|
||||||
python_push_type("pysocket_op", pykore, &pysocket_op_type);
|
python_push_type("pysocket_op", pykore, &pysocket_op_type);
|
||||||
|
@ -832,7 +842,7 @@ python_kore_task_create(PyObject *self, PyObject *args)
|
||||||
if (!PyCoro_CheckExact(obj))
|
if (!PyCoro_CheckExact(obj))
|
||||||
fatal("%s: object is not a coroutine", __func__);
|
fatal("%s: object is not a coroutine", __func__);
|
||||||
|
|
||||||
python_coro_create(obj);
|
python_coro_create(obj, NULL);
|
||||||
Py_INCREF(obj);
|
Py_INCREF(obj);
|
||||||
|
|
||||||
Py_RETURN_NONE;
|
Py_RETURN_NONE;
|
||||||
|
@ -906,6 +916,20 @@ python_kore_queue(PyObject *self, PyObject *args)
|
||||||
return ((PyObject *)queue);
|
return ((PyObject *)queue);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
python_kore_lock(PyObject *self, PyObject *args)
|
||||||
|
{
|
||||||
|
struct pylock *lock;
|
||||||
|
|
||||||
|
if ((lock = PyObject_New(struct pylock, &pylock_type)) == NULL)
|
||||||
|
return (NULL);
|
||||||
|
|
||||||
|
lock->owner = NULL;
|
||||||
|
TAILQ_INIT(&lock->ops);
|
||||||
|
|
||||||
|
return ((PyObject *)lock);
|
||||||
|
}
|
||||||
|
|
||||||
static PyObject *
|
static PyObject *
|
||||||
python_kore_fatal(PyObject *self, PyObject *args)
|
python_kore_fatal(PyObject *self, PyObject *args)
|
||||||
{
|
{
|
||||||
|
@ -1162,7 +1186,6 @@ pysocket_op_dealloc(struct pysocket_op *op)
|
||||||
kore_buf_cleanup(&op->data.buffer);
|
kore_buf_cleanup(&op->data.buffer);
|
||||||
|
|
||||||
Py_DECREF(op->data.socket);
|
Py_DECREF(op->data.socket);
|
||||||
Py_DECREF(op->data.coro->obj);
|
|
||||||
|
|
||||||
PyObject_Del((PyObject *)op);
|
PyObject_Del((PyObject *)op);
|
||||||
}
|
}
|
||||||
|
@ -1347,8 +1370,10 @@ pysocket_async_recv(struct pysocket_op *op)
|
||||||
ptr = (const char *)op->data.buffer.data;
|
ptr = (const char *)op->data.buffer.data;
|
||||||
|
|
||||||
bytes = PyBytes_FromStringAndSize(ptr, ret);
|
bytes = PyBytes_FromStringAndSize(ptr, ret);
|
||||||
if (bytes != NULL)
|
if (bytes != NULL) {
|
||||||
PyErr_SetObject(PyExc_StopIteration, bytes);
|
PyErr_SetObject(PyExc_StopIteration, bytes);
|
||||||
|
Py_DECREF(bytes);
|
||||||
|
}
|
||||||
|
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
|
@ -1548,6 +1573,149 @@ pyqueue_op_iternext(struct pyqueue_op *op)
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
pylock_dealloc(struct pylock *lock)
|
||||||
|
{
|
||||||
|
struct pylock_op *op;
|
||||||
|
|
||||||
|
while ((op = TAILQ_FIRST(&lock->ops)) != NULL) {
|
||||||
|
TAILQ_REMOVE(&lock->ops, op, list);
|
||||||
|
op->active = 0;
|
||||||
|
Py_DECREF((PyObject *)op);
|
||||||
|
}
|
||||||
|
|
||||||
|
PyObject_Del((PyObject *)op);
|
||||||
|
}
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
pylock_aenter(struct pylock *lock, PyObject *args)
|
||||||
|
{
|
||||||
|
struct pylock_op *op;
|
||||||
|
|
||||||
|
if (lock->owner != NULL && lock->owner->id == coro_running->id) {
|
||||||
|
PyErr_SetString(PyExc_RuntimeError, "recursive lock detected");
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((op = PyObject_New(struct pylock_op, &pylock_op_type)) == NULL)
|
||||||
|
return (NULL);
|
||||||
|
|
||||||
|
op->active = 1;
|
||||||
|
op->lock = lock;
|
||||||
|
op->locking = 1;
|
||||||
|
op->coro = coro_running;
|
||||||
|
|
||||||
|
Py_INCREF((PyObject *)op);
|
||||||
|
Py_INCREF((PyObject *)lock);
|
||||||
|
|
||||||
|
TAILQ_INSERT_TAIL(&lock->ops, op, list);
|
||||||
|
|
||||||
|
return ((PyObject *)op);
|
||||||
|
}
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
pylock_aexit(struct pylock *lock, PyObject *args)
|
||||||
|
{
|
||||||
|
struct pylock_op *op;
|
||||||
|
|
||||||
|
if (lock->owner == NULL || lock->owner->id != coro_running->id) {
|
||||||
|
PyErr_SetString(PyExc_RuntimeError, "invalid lock owner");
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((op = PyObject_New(struct pylock_op, &pylock_op_type)) == NULL)
|
||||||
|
return (NULL);
|
||||||
|
|
||||||
|
op->active = 1;
|
||||||
|
op->lock = lock;
|
||||||
|
op->locking = 0;
|
||||||
|
op->coro = coro_running;
|
||||||
|
|
||||||
|
Py_INCREF((PyObject *)op);
|
||||||
|
Py_INCREF((PyObject *)lock);
|
||||||
|
|
||||||
|
TAILQ_INSERT_TAIL(&lock->ops, op, list);
|
||||||
|
|
||||||
|
return ((PyObject *)op);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
pylock_do_release(struct pylock *lock)
|
||||||
|
{
|
||||||
|
struct pylock_op *op;
|
||||||
|
|
||||||
|
lock->owner = NULL;
|
||||||
|
|
||||||
|
TAILQ_FOREACH(op, &lock->ops, list) {
|
||||||
|
if (op->locking == 0)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
TAILQ_REMOVE(&op->lock->ops, op, list);
|
||||||
|
|
||||||
|
if (op->coro->request != NULL)
|
||||||
|
http_request_wakeup(op->coro->request);
|
||||||
|
else
|
||||||
|
python_coro_wakeup(op->coro);
|
||||||
|
|
||||||
|
op->active = 0;
|
||||||
|
Py_DECREF((PyObject *)op);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
pylock_op_dealloc(struct pylock_op *op)
|
||||||
|
{
|
||||||
|
if (op->active) {
|
||||||
|
TAILQ_REMOVE(&op->lock->ops, op, list);
|
||||||
|
op->active = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
Py_DECREF((PyObject *)op->lock);
|
||||||
|
PyObject_Del((PyObject *)op);
|
||||||
|
}
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
pylock_op_await(PyObject *obj)
|
||||||
|
{
|
||||||
|
Py_INCREF(obj);
|
||||||
|
return (obj);
|
||||||
|
}
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
pylock_op_iternext(struct pylock_op *op)
|
||||||
|
{
|
||||||
|
if (op->locking == 0) {
|
||||||
|
if (op->lock->owner == NULL) {
|
||||||
|
PyErr_SetString(PyExc_RuntimeError,
|
||||||
|
"no lock owner set");
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (op->lock->owner->id != coro_running->id) {
|
||||||
|
PyErr_SetString(PyExc_RuntimeError,
|
||||||
|
"lock not owned by caller");
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
pylock_do_release(op->lock);
|
||||||
|
} else {
|
||||||
|
if (op->lock->owner != NULL) {
|
||||||
|
Py_RETURN_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
op->lock->owner = coro_running;
|
||||||
|
}
|
||||||
|
|
||||||
|
op->active = 0;
|
||||||
|
TAILQ_REMOVE(&op->lock->ops, op, list);
|
||||||
|
PyErr_SetNone(PyExc_StopIteration);
|
||||||
|
|
||||||
|
Py_DECREF((PyObject *)op);
|
||||||
|
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
|
||||||
static PyObject *
|
static PyObject *
|
||||||
pyhttp_request_alloc(const struct http_request *req)
|
pyhttp_request_alloc(const struct http_request *req)
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue