mirror of https://git.kore.io/kore.git
Add kore.gather() to the python api.
Allows one to run coroutines concurrently and gather all their results in a single returned list. If any of the coroutines throw an exception the exception is returned as the value of that coroutine in the returned list.
This commit is contained in:
Joris Vink
parent
740acb4760
commit
1c30da855c
|
|
@ -22,12 +22,15 @@ struct python_coro {
|
|||
int state;
|
||||
PyObject *obj;
|
||||
struct pysocket_op *sockop;
|
||||
struct pygather_op *gatherop;
|
||||
struct http_request *request;
|
||||
PyObject *exception;
|
||||
char *exception_msg;
|
||||
TAILQ_ENTRY(python_coro) list;
|
||||
};
|
||||
|
||||
TAILQ_HEAD(coro_list, python_coro);
|
||||
|
||||
static PyObject *python_kore_log(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_lock(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_proc(PyObject *, PyObject *);
|
||||
|
|
@ -35,6 +38,7 @@ static PyObject *python_kore_bind(PyObject *, PyObject *);
|
|||
static PyObject *python_kore_timer(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_fatal(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_queue(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_gather(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_fatalx(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_suspend(PyObject *, PyObject *);
|
||||
static PyObject *python_kore_shutdown(PyObject *, PyObject *);
|
||||
|
|
@ -60,6 +64,7 @@ static struct PyMethodDef pykore_methods[] = {
|
|||
METHOD("bind", python_kore_bind, METH_VARARGS),
|
||||
METHOD("timer", python_kore_timer, METH_VARARGS),
|
||||
METHOD("queue", python_kore_queue, METH_VARARGS),
|
||||
METHOD("gather", python_kore_gather, METH_VARARGS),
|
||||
METHOD("fatal", python_kore_fatal, METH_VARARGS),
|
||||
METHOD("fatalx", python_kore_fatalx, METH_VARARGS),
|
||||
METHOD("suspend", python_kore_suspend, METH_VARARGS),
|
||||
|
|
@ -415,6 +420,47 @@ static PyTypeObject pyproc_op_type = {
|
|||
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
|
||||
};
|
||||
|
||||
struct pygather_coro {
|
||||
struct python_coro *coro;
|
||||
PyObject *result;
|
||||
TAILQ_ENTRY(pygather_coro) list;
|
||||
};
|
||||
|
||||
struct pygather_result {
|
||||
PyObject *obj;
|
||||
TAILQ_ENTRY(pygather_result) list;
|
||||
};
|
||||
|
||||
struct pygather_op {
|
||||
PyObject_HEAD
|
||||
int count;
|
||||
struct python_coro *coro;
|
||||
TAILQ_HEAD(, pygather_result) results;
|
||||
TAILQ_HEAD(, pygather_coro) coroutines;
|
||||
};
|
||||
|
||||
static void pygather_op_dealloc(struct pygather_op *);
|
||||
|
||||
static PyObject *pygather_op_await(PyObject *);
|
||||
static PyObject *pygather_op_iternext(struct pygather_op *);
|
||||
|
||||
static PyAsyncMethods pygather_op_async = {
|
||||
(unaryfunc)pygather_op_await,
|
||||
NULL,
|
||||
NULL
|
||||
};
|
||||
|
||||
static PyTypeObject pygather_op_type = {
|
||||
PyVarObject_HEAD_INIT(NULL, 0)
|
||||
.tp_name = "kore.pygather_op",
|
||||
.tp_doc = "coroutine gathering",
|
||||
.tp_as_async = &pygather_op_async,
|
||||
.tp_iternext = (iternextfunc)pygather_op_iternext,
|
||||
.tp_basicsize = sizeof(struct pygather_op),
|
||||
.tp_dealloc = (destructor)pygather_op_dealloc,
|
||||
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
|
||||
};
|
||||
|
||||
struct pyconnection {
|
||||
PyObject_HEAD
|
||||
struct connection *c;
|
||||
|
|
|
|||
222
src/python.c
222
src/python.c
|
|
@ -61,6 +61,9 @@ static void pytimer_run(void *, u_int64_t);
|
|||
static void pyproc_timeout(void *, u_int64_t);
|
||||
static void pysuspend_wakeup(void *, u_int64_t);
|
||||
|
||||
static void pygather_reap_coro(struct pygather_op *,
|
||||
struct python_coro *);
|
||||
|
||||
#if defined(KORE_USE_PGSQL)
|
||||
static PyObject *pykore_pgsql_alloc(struct http_request *,
|
||||
const char *, const char *);
|
||||
|
|
@ -153,17 +156,18 @@ static TAILQ_HEAD(, pyproc) procs;
|
|||
|
||||
static struct kore_pool coro_pool;
|
||||
static struct kore_pool queue_wait_pool;
|
||||
static struct kore_pool gather_coro_pool;
|
||||
static struct kore_pool queue_object_pool;
|
||||
static struct kore_pool gather_result_pool;
|
||||
|
||||
static u_int64_t coro_id;
|
||||
static int coro_count;
|
||||
static TAILQ_HEAD(, python_coro) coro_runnable;
|
||||
static TAILQ_HEAD(, python_coro) coro_suspended;
|
||||
static struct coro_list coro_runnable;
|
||||
static struct coro_list coro_suspended;
|
||||
|
||||
extern const char *__progname;
|
||||
|
||||
/* XXX */
|
||||
static struct http_request *req_running = NULL;
|
||||
static struct python_coro *coro_running = NULL;
|
||||
|
||||
void
|
||||
|
|
@ -180,8 +184,12 @@ kore_python_init(void)
|
|||
|
||||
kore_pool_init(&queue_wait_pool, "queue_wait_pool",
|
||||
sizeof(struct pyqueue_waiting), 100);
|
||||
kore_pool_init(&gather_coro_pool, "gather_coro_pool",
|
||||
sizeof(struct pygather_coro), 100);
|
||||
kore_pool_init(&queue_object_pool, "queue_object_pool",
|
||||
sizeof(struct pyqueue_object), 100);
|
||||
kore_pool_init(&gather_result_pool, "gather_result_pool",
|
||||
sizeof(struct pygather_result), 100);
|
||||
|
||||
PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocator);
|
||||
PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocator);
|
||||
|
|
@ -212,14 +220,27 @@ kore_python_path(const char *path)
|
|||
void
|
||||
kore_python_coro_run(void)
|
||||
{
|
||||
struct pygather_op *op;
|
||||
struct python_coro *coro, *next;
|
||||
|
||||
for (coro = TAILQ_FIRST(&coro_runnable); coro != NULL; coro = next) {
|
||||
next = TAILQ_NEXT(coro, list);
|
||||
|
||||
if (coro->state != CORO_STATE_RUNNABLE)
|
||||
fatal("non-runnable coro on coro_runnable");
|
||||
if (python_coro_run(coro) == KORE_RESULT_OK)
|
||||
kore_python_coro_delete(coro);
|
||||
|
||||
if (python_coro_run(coro) == KORE_RESULT_OK) {
|
||||
if (coro->gatherop != NULL) {
|
||||
op = coro->gatherop;
|
||||
if (op->coro->request != NULL)
|
||||
http_request_wakeup(op->coro->request);
|
||||
else
|
||||
python_coro_wakeup(op->coro);
|
||||
pygather_reap_coro(op, coro);
|
||||
} else {
|
||||
kore_python_coro_delete(coro);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -268,14 +289,28 @@ kore_python_log_error(const char *function)
|
|||
return;
|
||||
}
|
||||
|
||||
if ((repr = PyObject_Repr(value)) == NULL)
|
||||
sval = "unknown";
|
||||
else
|
||||
sval = PyUnicode_AsUTF8(repr);
|
||||
if (value == NULL || !PyObject_IsInstance(value, type))
|
||||
PyErr_NormalizeException(&type, &value, &traceback);
|
||||
|
||||
kore_log(LOG_ERR, "uncaught exception %s in '%s'", sval, function);
|
||||
/*
|
||||
* If we're in an active coroutine and it was tied to a gather
|
||||
* operation we have to make sure we can use the Exception that
|
||||
* was thrown as the result value so we can propagate it via the
|
||||
* return list of kore.gather().
|
||||
*/
|
||||
if (coro_running != NULL && coro_running->gatherop != NULL) {
|
||||
PyErr_SetObject(PyExc_StopIteration, value);
|
||||
} else {
|
||||
if ((repr = PyObject_Repr(value)) == NULL)
|
||||
sval = "unknown";
|
||||
else
|
||||
sval = PyUnicode_AsUTF8(repr);
|
||||
|
||||
Py_XDECREF(repr);
|
||||
kore_log(LOG_ERR,
|
||||
"uncaught exception %s in '%s'", sval, function);
|
||||
|
||||
Py_XDECREF(repr);
|
||||
}
|
||||
|
||||
Py_DECREF(type);
|
||||
Py_DECREF(value);
|
||||
|
|
@ -391,6 +426,7 @@ python_coro_create(PyObject *obj, struct http_request *req)
|
|||
coro_count++;
|
||||
|
||||
coro->sockop = NULL;
|
||||
coro->gatherop = NULL;
|
||||
coro->exception = NULL;
|
||||
coro->exception_msg = NULL;
|
||||
|
||||
|
|
@ -482,17 +518,13 @@ python_runtime_http_request(void *addr, struct http_request *req)
|
|||
{
|
||||
PyObject *pyret, *pyreq, *args, *callable;
|
||||
|
||||
req_running = req;
|
||||
|
||||
if (req->py_coro != NULL) {
|
||||
python_coro_wakeup(req->py_coro);
|
||||
if (python_coro_run(req->py_coro) == KORE_RESULT_OK) {
|
||||
kore_python_coro_delete(req->py_coro);
|
||||
req->py_coro = NULL;
|
||||
req_running = NULL;
|
||||
return (KORE_RESULT_OK);
|
||||
}
|
||||
req_running = NULL;
|
||||
return (KORE_RESULT_RETRY);
|
||||
}
|
||||
|
||||
|
|
@ -518,16 +550,13 @@ python_runtime_http_request(void *addr, struct http_request *req)
|
|||
}
|
||||
|
||||
if (PyCoro_CheckExact(pyret)) {
|
||||
http_request_sleep(req);
|
||||
req->py_coro = python_coro_create(pyret, req);
|
||||
req_running = NULL;
|
||||
/* XXX merge with the above python_coro_run() block. */
|
||||
if (python_coro_run(req->py_coro) == KORE_RESULT_OK) {
|
||||
kore_python_coro_delete(req->py_coro);
|
||||
req->py_coro = NULL;
|
||||
req_running = NULL;
|
||||
return (KORE_RESULT_OK);
|
||||
}
|
||||
http_request_sleep(req);
|
||||
return (KORE_RESULT_RETRY);
|
||||
}
|
||||
|
||||
|
|
@ -535,7 +564,6 @@ python_runtime_http_request(void *addr, struct http_request *req)
|
|||
fatal("python_runtime_http_request: unexpected return type");
|
||||
|
||||
Py_DECREF(pyret);
|
||||
req_running = NULL;
|
||||
|
||||
return (KORE_RESULT_OK);
|
||||
}
|
||||
|
|
@ -969,6 +997,59 @@ python_kore_queue(PyObject *self, PyObject *args)
|
|||
return ((PyObject *)queue);
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
python_kore_gather(PyObject *self, PyObject *args)
|
||||
{
|
||||
struct pygather_op *op;
|
||||
PyObject *obj;
|
||||
struct pygather_coro *coro;
|
||||
Py_ssize_t sz, idx;
|
||||
|
||||
if (coro_running == NULL) {
|
||||
PyErr_SetString(PyExc_RuntimeError,
|
||||
"kore.gather only available in coroutines");
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
sz = PyTuple_Size(args);
|
||||
|
||||
if (sz > INT_MAX) {
|
||||
PyErr_SetString(PyExc_TypeError, "too many arguments");
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
op = PyObject_New(struct pygather_op, &pygather_op_type);
|
||||
if (op == NULL)
|
||||
return (NULL);
|
||||
|
||||
op->count = (int)sz;
|
||||
op->coro = coro_running;
|
||||
|
||||
TAILQ_INIT(&op->results);
|
||||
TAILQ_INIT(&op->coroutines);
|
||||
|
||||
for (idx = 0; idx < sz; idx++) {
|
||||
if ((obj = PyTuple_GetItem(args, idx)) == NULL)
|
||||
return (NULL);
|
||||
|
||||
if (!PyCoro_CheckExact(obj)) {
|
||||
PyErr_SetString(PyExc_TypeError, "not a coroutine");
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
Py_INCREF(obj);
|
||||
|
||||
coro = kore_pool_get(&gather_coro_pool);
|
||||
|
||||
coro->coro = python_coro_create(obj, NULL);
|
||||
coro->coro->gatherop = op;
|
||||
|
||||
TAILQ_INSERT_TAIL(&op->coroutines, coro, list);
|
||||
}
|
||||
|
||||
return ((PyObject *)op);
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
python_kore_lock(PyObject *self, PyObject *args)
|
||||
{
|
||||
|
|
@ -2285,6 +2366,107 @@ pyproc_op_iternext(struct pyproc_op *op)
|
|||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
pygather_reap_coro(struct pygather_op *op, struct python_coro *reap)
|
||||
{
|
||||
struct pygather_coro *coro;
|
||||
struct pygather_result *result;
|
||||
|
||||
TAILQ_FOREACH(coro, &op->coroutines, list) {
|
||||
if (coro->coro->id == reap->id)
|
||||
break;
|
||||
}
|
||||
|
||||
if (coro == NULL)
|
||||
fatal("coroutine %" PRIu64 " not found in gather", reap->id);
|
||||
|
||||
result = kore_pool_get(&gather_result_pool);
|
||||
result->obj = NULL;
|
||||
|
||||
if (_PyGen_FetchStopIterationValue(&result->obj) == -1) {
|
||||
result->obj = Py_None;
|
||||
Py_INCREF(Py_None);
|
||||
}
|
||||
|
||||
TAILQ_INSERT_TAIL(&op->results, result, list);
|
||||
|
||||
TAILQ_REMOVE(&op->coroutines, coro, list);
|
||||
kore_pool_put(&gather_coro_pool, coro);
|
||||
|
||||
kore_python_coro_delete(reap);
|
||||
}
|
||||
|
||||
static void
|
||||
pygather_op_dealloc(struct pygather_op *op)
|
||||
{
|
||||
struct pygather_coro *coro, *next;
|
||||
struct pygather_result *res, *rnext;
|
||||
|
||||
for (coro = TAILQ_FIRST(&op->coroutines); coro != NULL; coro = next) {
|
||||
next = TAILQ_NEXT(coro, list);
|
||||
TAILQ_REMOVE(&op->coroutines, coro, list);
|
||||
|
||||
/* Make sure we don't end up in pygather_reap_coro(). */
|
||||
coro->coro->gatherop = NULL;
|
||||
|
||||
kore_python_coro_delete(coro->coro);
|
||||
kore_pool_put(&gather_coro_pool, coro);
|
||||
}
|
||||
|
||||
for (res = TAILQ_FIRST(&op->results); res != NULL; res = rnext) {
|
||||
rnext = TAILQ_NEXT(res, list);
|
||||
TAILQ_REMOVE(&op->results, res, list);
|
||||
|
||||
Py_DECREF(res->obj);
|
||||
kore_pool_put(&gather_result_pool, res);
|
||||
}
|
||||
|
||||
PyObject_Del((PyObject *)op);
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
pygather_op_await(PyObject *obj)
|
||||
{
|
||||
Py_INCREF(obj);
|
||||
return (obj);
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
pygather_op_iternext(struct pygather_op *op)
|
||||
{
|
||||
int idx;
|
||||
struct pygather_result *res, *next;
|
||||
PyObject *list, *obj;
|
||||
|
||||
if (!TAILQ_EMPTY(&op->coroutines)) {
|
||||
Py_RETURN_NONE;
|
||||
}
|
||||
|
||||
if ((list = PyList_New(op->count)) == NULL)
|
||||
return (NULL);
|
||||
|
||||
idx = 0;
|
||||
|
||||
for (res = TAILQ_FIRST(&op->results); res != NULL; res = next) {
|
||||
next = TAILQ_NEXT(res, list);
|
||||
TAILQ_REMOVE(&op->results, res, list);
|
||||
|
||||
obj = res->obj;
|
||||
res->obj = NULL;
|
||||
kore_pool_put(&gather_result_pool, res);
|
||||
|
||||
if (PyList_SetItem(list, idx++, obj) != 0) {
|
||||
Py_DECREF(list);
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
|
||||
PyErr_SetObject(PyExc_StopIteration, list);
|
||||
Py_DECREF(list);
|
||||
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
pyhttp_request_alloc(const struct http_request *req)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue