diff --git a/Makefile b/Makefile index ffae5c8..16c401d 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,16 @@ ifneq ("$(PYTHON)", "") FEATURES_INC+=$(KORE_PYTHON_INC) endif +ifneq ("$(CURL)", "") + S_SRC+=src/curl.c + KORE_CURL_LIB?=$(shell pkg-config --libs libcurl) + KORE_CURL_INC?=$(shell pkg-config --cflags libcurl) + LDFLAGS+=$(KORE_CURL_LIB) + CFLAGS+=$(KORE_CURL_INC) -DKORE_USE_CURL + FEATURES+=-DKORE_USE_CURL + FEATURES_INC+=$(KORE_CURL_INC) +endif + ifneq ("$(SANITIZE)", "") CFLAGS+=-fsanitize=$(SANITIZE) LDFLAGS+=-fsanitize=$(SANITIZE) diff --git a/examples/async-curl/.gitignore b/examples/async-curl/.gitignore new file mode 100644 index 0000000..2c491bc --- /dev/null +++ b/examples/async-curl/.gitignore @@ -0,0 +1,6 @@ +*.o +.flavor +.objs +ht.so +assets.h +cert diff --git a/examples/async-curl/README.md b/examples/async-curl/README.md new file mode 100644 index 0000000..435ea93 --- /dev/null +++ b/examples/async-curl/README.md @@ -0,0 +1,12 @@ +Kore asynchronous libcurl integration example. + +This example demonstrates how you can use the asynchronous libcurl +api from Kore to perform HTTP client requests, or FTP requests, or send +emails all in an asynchronous fashion. + +Run: +``` + $ kodev run + $ curl https://127.0.0.1:8888 + $ curl https://127.0.0.1:8888/ftp +``` diff --git a/examples/async-curl/conf/async-curl.conf b/examples/async-curl/conf/async-curl.conf new file mode 100644 index 0000000..7833894 --- /dev/null +++ b/examples/async-curl/conf/async-curl.conf @@ -0,0 +1,14 @@ +# ht configuration + +bind 127.0.0.1 8888 + +workers 1 +tls_dhparam dh2048.pem + +domain * { + certfile cert/server.pem + certkey cert/key.pem + + static / http + static /ftp ftp +} diff --git a/examples/async-curl/conf/build.conf b/examples/async-curl/conf/build.conf new file mode 100644 index 0000000..34362d0 --- /dev/null +++ b/examples/async-curl/conf/build.conf @@ -0,0 +1,34 @@ +# ht build config +# You can switch flavors using: kodev flavor [newflavor] + +# Set to yes if you wish to produce a single binary instead +# of a dynamic library. If you set this to yes you must also +# set kore_source together with kore_flavor. +single_binary=yes +kore_source=../../ +kore_flavor=CURL=1 + +# The flags below are shared between flavors +cflags=-Wall -Wmissing-declarations -Wshadow +cflags=-Wstrict-prototypes -Wmissing-prototypes +cflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +cxxflags=-Wall -Wmissing-declarations -Wshadow +cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +# Mime types for assets served via the builtin asset_serve_* +#mime_add=txt:text/plain; charset=utf-8 +#mime_add=png:image/png +#mime_add=html:text/html; charset=utf-8 + +dev { + # These flags are added to the shared ones when + # you build the "dev" flavor. + cflags=-g + cxxflags=-g +} + +#prod { +# You can specify additional flags here which are only +# included if you build with the "prod" flavor. +#} diff --git a/examples/async-curl/src/ftp.c b/examples/async-curl/src/ftp.c new file mode 100644 index 0000000..33ddd7f --- /dev/null +++ b/examples/async-curl/src/ftp.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * This example is the same as the HTTP one (see src/http.c) except + * we fetch an FTP URL. + */ + +#include +#include +#include + +int ftp(struct http_request *); + +static int state_setup(struct http_request *); +static int state_result(struct http_request *); + +static struct http_state states[] = { + KORE_HTTP_STATE(state_setup), + KORE_HTTP_STATE(state_result) +}; + +int +ftp(struct http_request *req) +{ + return (http_state_run(states, 2, req)); +} + +static int +state_setup(struct http_request *req) +{ + struct kore_curl *client; + + client = http_state_create(req, sizeof(*client)); + + if (!kore_curl_init(client, + "http://ftp.eu.openbsd.org/pub/OpenBSD/README")) { + http_response(req, 500, NULL, 0); + return (HTTP_STATE_COMPLETE); + } + + kore_curl_bind_request(client, req); + kore_curl_run(client); + + req->fsm_state = 1; + return (HTTP_STATE_RETRY); +} + +static int +state_result(struct http_request *req) +{ + size_t len; + const u_int8_t *body; + struct kore_curl *client; + + client = http_state_get(req); + + if (!kore_curl_success(client)) { + kore_curl_logerror(client); + http_response(req, 500, NULL, 0); + } else { + kore_curl_response_as_bytes(client, &body, &len); + http_response(req, HTTP_STATUS_OK, body, len); + } + + kore_curl_cleanup(client); + + return (HTTP_STATE_COMPLETE); +} diff --git a/examples/async-curl/src/http.c b/examples/async-curl/src/http.c new file mode 100644 index 0000000..9b57af5 --- /dev/null +++ b/examples/async-curl/src/http.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * This example demonstrates how easy it is to perform asynchronous + * HTTP client requests using the integrated libcurl support. + * + * In this example we setup 2 states for an HTTP request: + * 1) setup + * We initialize the HTTP request and fire it off. + * This will put our HTTP request to sleep and it be woken up + * by Kore when a response is available or something went wrong. + * + * 2) result + * After we have woken up we have access to the result. + */ + +#include +#include +#include + +int http(struct http_request *); + +static int state_setup(struct http_request *); +static int state_result(struct http_request *); + +/* Our states. */ +static struct http_state states[] = { + KORE_HTTP_STATE(state_setup), + KORE_HTTP_STATE(state_result) +}; + +/* Transcend into the HTTP state machine for a request. */ +int +http(struct http_request *req) +{ + return (http_state_run(states, 2, req)); +} + +/* + * Setup the HTTP client request using the integrated curl API and the easy + * to use HTTP client api. + */ +static int +state_setup(struct http_request *req) +{ + struct kore_curl *client; + + client = http_state_create(req, sizeof(*client)); + + /* Initialize curl. */ + if (!kore_curl_init(client, "https://kore.io")) { + http_response(req, 500, NULL, 0); + return (HTTP_STATE_COMPLETE); + } + + /* Setup our HTTP client request. */ + kore_curl_http_setup(client, HTTP_METHOD_GET, NULL, 0); + + /* Add some headers. */ + kore_curl_http_set_header(client, "x-source", "from-example"); + + /* We could opt to override some settings ourselves if we wanted. */ + /* curl_easy_setopt(client->handle, CURLOPT_SSL_VERIFYHOST, 0); */ + /* curl_easy_setopt(client->handle, CURLOPT_SSL_VERIFYPEER, 0); */ + + /* + * Bind the HTTP client request to our HTTP request so we get woken + * up once a response is available. + */ + kore_curl_bind_request(client, req); + + /* + * Now fire off the request onto the event loop. + * This will put us to sleep. + */ + kore_curl_run(client); + + /* Make sure we go onto the next state once woken up. */ + req->fsm_state = 1; + + /* Tell Kore we can't complete this immediately. */ + return (HTTP_STATE_RETRY); +} + +/* + * This state is called when a result for the HTTP request call is + * available to us. + */ +static int +state_result(struct http_request *req) +{ + size_t len; + const u_int8_t *body; + const char *header; + struct kore_curl *client; + + /* Get the state attached to the HTTP request. */ + client = http_state_get(req); + + /* Check if we were succesfull, if not log an error. */ + if (!kore_curl_success(client)) { + kore_curl_logerror(client); + http_response(req, 500, NULL, 0); + } else { + /* + * Success! We now have the body available to us. + */ + kore_curl_response_as_bytes(client, &body, &len); + + /* We could check the existance of a header: */ + if (kore_curl_http_get_header(client, "server", &header)) + printf("got server header: '%s'\n", header); + + /* + * Respond to our client with the status and body from + * the HTTP client request we did. + */ + http_response(req, client->http.status, body, len); + } + + /* Cleanup. */ + kore_curl_cleanup(client); + + /* State is now finished. */ + return (HTTP_STATE_COMPLETE); +} diff --git a/examples/python-async/README.md b/examples/python-async/README.md index 73b97bd..97451e3 100644 --- a/examples/python-async/README.md +++ b/examples/python-async/README.md @@ -1,5 +1,8 @@ Kore python async/await examples. +This example also shows off the asynchronous HTTP client support +and requires libcurl on your machine. + Run: ``` $ kodev run @@ -11,4 +14,5 @@ Test: $ curl -k http://127.0.0.1:8888/lock $ curl -k http://127.0.0.1:8888/proc $ curl -k http://127.0.0.1:8888/socket + $ curl -k http://127.0.0.1:8888/httpclient ``` diff --git a/examples/python-async/conf/build.conf b/examples/python-async/conf/build.conf index 3b3aafd..9d19b9c 100644 --- a/examples/python-async/conf/build.conf +++ b/examples/python-async/conf/build.conf @@ -6,7 +6,7 @@ # set kore_source together with kore_flavor. single_binary=yes kore_source=../../ -kore_flavor=PYTHON=1 NOTLS=1 +kore_flavor=PYTHON=1 CURL=1 NOTLS=1 # The flags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff --git a/examples/python-async/conf/python-async.conf b/examples/python-async/conf/python-async.conf index e9abb0b..7076151 100644 --- a/examples/python-async/conf/python-async.conf +++ b/examples/python-async/conf/python-async.conf @@ -9,6 +9,7 @@ python_import ./src/async_lock.py python_import ./src/async_queue.py python_import ./src/async_process.py python_import ./src/async_socket.py +python_import ./src/async_http.py domain * { static /queue async_queue @@ -17,4 +18,6 @@ domain * { static /socket async_socket static /socket-test socket_test + + static /httpclient httpclient } diff --git a/examples/python-async/src/async_http.py b/examples/python-async/src/async_http.py new file mode 100644 index 0000000..9e63e47 --- /dev/null +++ b/examples/python-async/src/async_http.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2019 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +# +# Asynchronous HTTP client example. +# + +import kore + +# Handler called for /httpclient +async def httpclient(req): + # Create an httpclient. + client = kore.httpclient("https://kore.io") + + # Do a simple GET request. + status, body = await client.get() + print("status: %d, body: '%s'" % (status, body)) + + # Reuse and perform another GET request, returning headers too this time. + status, headers, body = await client.get(return_headers=True) + print("status: %d, headers: '%s'" % (status, headers)) + + # What happens if we post something? + status, body = await client.post(body=b"hello world") + print("status: %d, body: '%s'" % (status, body)) + + # Add some custom headers to our requests. + status, body = await client.get( + headers={ + "x-my-header": "async-http" + } + ) + + req.response(200, b'async done') diff --git a/include/kore/curl.h b/include/kore/curl.h new file mode 100644 index 0000000..0b7314e --- /dev/null +++ b/include/kore/curl.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __H_CURL_H +#define __H_CURL_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include + +#include "http.h" + +#define KORE_CURL_TIMEOUT 60 +#define KORE_CURL_RECV_MAX (1024 * 1024 * 2) + +#define KORE_CURL_STATUS_UNKNOWN 0 +#define KORE_CURL_FLAG_HTTP_PARSED_HEADERS 0x0001 +#define KORE_CURL_FLAG_BOUND 0x0002 + +#define KORE_CURL_TYPE_CUSTOM 1 +#define KORE_CURL_TYPE_HTTP_CLIENT 2 + +struct kore_curl { + int type; + int flags; + CURLcode result; + + char *url; + CURL *handle; + struct kore_buf *response; + + struct http_request *req; + void *arg; + void (*cb)(struct kore_curl *, void *); + + char errbuf[CURL_ERROR_SIZE]; + + /* For the simplified HTTP api. */ + struct { + int status; + struct curl_slist *hdrlist; + + struct kore_buf *tosend; + struct kore_buf *headers; + + TAILQ_HEAD(, http_header) resp_hdrs; + } http; + + LIST_ENTRY(kore_curl) list; +}; + +extern u_int16_t kore_curl_timeout; +extern u_int64_t kore_curl_recv_max; + +void kore_curl_sysinit(void); +void kore_curl_run(struct kore_curl *); +void kore_curl_cleanup(struct kore_curl *); +int kore_curl_success(struct kore_curl *); +void kore_curl_logerror(struct kore_curl *); +int kore_curl_init(struct kore_curl *, const char *); + +size_t kore_curl_tobuf(char *, size_t, size_t, void *); +size_t kore_curl_frombuf(char *, size_t, size_t, void *); + +void kore_curl_http_parse_headers(struct kore_curl *); +void kore_curl_http_set_header(struct kore_curl *, const char *, + const char *); +int kore_curl_http_get_header(struct kore_curl *, const char *, + const char **); +void kore_curl_http_setup(struct kore_curl *, int, const void *, size_t); + +char *kore_curl_response_as_string(struct kore_curl *); +void kore_curl_response_as_bytes(struct kore_curl *, + const u_int8_t **, size_t *); + +void kore_curl_bind_request(struct kore_curl *, struct http_request *); +void kore_curl_bind_callback(struct kore_curl *, + void (*cb)(struct kore_curl *, void *), void *); + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/include/kore/http.h b/include/kore/http.h index b19c143..e02dd93 100644 --- a/include/kore/http.h +++ b/include/kore/http.h @@ -221,6 +221,7 @@ struct http_file { #define HTTP_BODY_DIGEST_STRLEN ((HTTP_BODY_DIGEST_LEN * 2) + 1) struct kore_task; +struct http_client; struct http_request { u_int8_t method; @@ -254,8 +255,17 @@ struct http_request { u_int8_t http_body_digest[HTTP_BODY_DIGEST_LEN]; +#if defined(KORE_USE_CURL) + LIST_HEAD(, kore_curl) chandles; +#endif + +#if defined(KORE_USE_TASKS) LIST_HEAD(, kore_task) tasks; +#endif + +#if defined(KORE_USE_PGSQL) LIST_HEAD(, kore_pgsql) pgsqls; +#endif TAILQ_HEAD(, http_cookie) req_cookies; TAILQ_HEAD(, http_cookie) resp_cookies; @@ -291,6 +301,7 @@ extern u_int32_t http_request_limit; extern u_int32_t http_request_count; extern u_int64_t http_body_disk_offload; extern char *http_body_disk_path; +extern struct kore_pool http_header_pool; void kore_accesslog(struct http_request *); @@ -302,6 +313,7 @@ void http_process(void); const char *http_status_text(int); const char *http_method_text(int); time_t http_date_to_time(char *); +char *http_validate_header(char *); void http_request_free(struct http_request *); void http_request_sleep(struct http_request *); void http_request_wakeup(struct http_request *); diff --git a/include/kore/kore.h b/include/kore/kore.h index 8e9eab5..4a97099 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -164,6 +164,7 @@ TAILQ_HEAD(netbuf_head, netbuf); #define KORE_TYPE_PGSQL_CONN 3 #define KORE_TYPE_TASK 4 #define KORE_TYPE_PYSOCKET 5 +#define KORE_TYPE_CURL_HANDLE 6 #define CONN_STATE_UNKNOWN 0 #define CONN_STATE_TLS_SHAKE 1 diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index ec6633b..5784963 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -53,6 +53,11 @@ static PyObject *python_kore_gather(PyObject *, PyObject *, PyObject *); static PyObject *python_kore_pgsql_register(PyObject *, PyObject *); #endif +#if defined(KORE_USE_CURL) +static PyObject *python_kore_httpclient(PyObject *, + PyObject *, PyObject *); +#endif + static PyObject *python_websocket_broadcast(PyObject *, PyObject *); #define METHOD(n, c, a) { n, (PyCFunction)c, a, NULL } @@ -80,6 +85,10 @@ static struct PyMethodDef pykore_methods[] = { METHOD("websocket_broadcast", python_websocket_broadcast, METH_VARARGS), #if defined(KORE_USE_PGSQL) METHOD("register_database", python_kore_pgsql_register, METH_VARARGS), +#endif +#if defined(KORE_USE_CURL) + METHOD("httpclient", python_kore_httpclient, + METH_VARARGS | METH_KEYWORDS), #endif { NULL, NULL, 0, NULL } }; @@ -639,6 +648,85 @@ static PyTypeObject pyhttp_file_type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, }; +#if defined(KORE_USE_CURL) +struct pyhttp_client { + PyObject_HEAD + char *url; + char *tlscert; + char *tlskey; +}; + +#define PYHTTP_CLIENT_OP_RUN 1 +#define PYHTTP_CLIENT_OP_RESULT 2 + +struct pyhttp_client_op { + PyObject_HEAD + int state; + int headers; + struct kore_curl curl; + struct python_coro *coro; +}; + +static PyObject *pyhttp_client_op_await(PyObject *); +static PyObject *pyhttp_client_op_iternext(struct pyhttp_client_op *); + +static void pyhttp_client_dealloc(struct pyhttp_client *); +static void pyhttp_client_op_dealloc(struct pyhttp_client_op *); + +static PyObject *pyhttp_client_get(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_put(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_post(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_head(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_patch(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_delete(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_options(struct pyhttp_client *, + PyObject *, PyObject *); + +static PyMethodDef pyhttp_client_methods[] = { + METHOD("get", pyhttp_client_get, METH_VARARGS | METH_KEYWORDS), + METHOD("put", pyhttp_client_put, METH_VARARGS | METH_KEYWORDS), + METHOD("post", pyhttp_client_post, METH_VARARGS | METH_KEYWORDS), + METHOD("head", pyhttp_client_head, METH_VARARGS | METH_KEYWORDS), + METHOD("patch", pyhttp_client_patch, METH_VARARGS | METH_KEYWORDS), + METHOD("delete", pyhttp_client_delete, METH_VARARGS | METH_KEYWORDS), + METHOD("options", pyhttp_client_options, METH_VARARGS | METH_KEYWORDS), + METHOD(NULL, NULL, -1) +}; + +static PyTypeObject pyhttp_client_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.httpclient", + .tp_doc = "An asynchronous HTTP client", + .tp_methods = pyhttp_client_methods, + .tp_basicsize = sizeof(struct pyhttp_client), + .tp_dealloc = (destructor)pyhttp_client_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +static PyAsyncMethods pyhttp_client_op_async = { + (unaryfunc)pyhttp_client_op_await, + NULL, + NULL +}; + +static PyTypeObject pyhttp_client_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.httpclientop", + .tp_doc = "Asynchronous HTTP client operation", + .tp_as_async = &pyhttp_client_op_async, + .tp_iternext = (iternextfunc)pyhttp_client_op_iternext, + .tp_basicsize = sizeof(struct pyhttp_client_op), + .tp_dealloc = (destructor)pyhttp_client_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; +#endif + #if defined(KORE_USE_PGSQL) #define PYKORE_PGSQL_PREINIT 1 diff --git a/src/config.c b/src/config.c index 219f808..71ec191 100644 --- a/src/config.c +++ b/src/config.c @@ -41,6 +41,10 @@ #include "python_api.h" #endif +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + /* XXX - This is becoming a clusterfuck. Fix it. */ static int configure_load(char *); @@ -129,6 +133,11 @@ static int configure_python_path(char *); static int configure_python_import(char *); #endif +#if defined(KORE_USE_CURL) +static int configure_curl_timeout(char *); +static int configure_curl_recv_max(char *); +#endif + static struct { const char *name; int (*configure)(char *); @@ -204,6 +213,10 @@ static struct { #endif #if defined(KORE_USE_TASKS) { "task_threads", configure_task_threads }, +#endif +#if defined(KORE_USE_CURL) + { "curl_timeout", configure_curl_timeout }, + { "curl_recv_max", configure_curl_recv_max }, #endif { NULL, NULL }, }; @@ -1475,3 +1488,33 @@ configure_add_pledge(char *pledge) return (KORE_RESULT_OK); } #endif + +#if defined(KORE_USE_CURL) +static int +configure_curl_recv_max(char *option) +{ + int err; + + kore_curl_recv_max = kore_strtonum64(option, 1, &err); + if (err != KORE_RESULT_OK) { + printf("bad curl_recv_max value: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int +configure_curl_timeout(char *option) +{ + int err; + + kore_curl_timeout = kore_strtonum(option, 10, 0, USHRT_MAX, &err); + if (err != KORE_RESULT_OK) { + printf("bad kore_curl_timeout value: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} +#endif diff --git a/src/curl.c b/src/curl.c new file mode 100644 index 0000000..7af77b0 --- /dev/null +++ b/src/curl.c @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include + +#include "kore.h" +#include "http.h" +#include "curl.h" + +#define FD_CACHE_BUCKETS 2048 + +struct fd_cache { + struct kore_event evt; + int fd; + int scheduled; + LIST_ENTRY(fd_cache) list; +}; + +static void curl_process(void); +static void curl_event_handle(void *, int); +static int curl_timer(CURLM *, long, void *); +static int curl_socket(CURL *, curl_socket_t, int, void *, void *); + +static struct fd_cache *fd_cache_get(int); + +static int running = 0; +static CURLM *multi = NULL; +static struct kore_timer *timer = NULL; +static struct kore_pool fd_cache_pool; +static LIST_HEAD(, fd_cache) cache[FD_CACHE_BUCKETS]; + +u_int16_t kore_curl_timeout = KORE_CURL_TIMEOUT; +u_int64_t kore_curl_recv_max = KORE_CURL_RECV_MAX; + +void +kore_curl_sysinit(void) +{ + int i; + CURLMcode res; + + if (curl_global_init(CURL_GLOBAL_ALL)) + fatal("failed to initialize libcurl"); + + if ((multi = curl_multi_init()) == NULL) + fatal("curl_multi_init(): failed"); + + /* XXX - make configurable? */ + curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, 500); + + if ((res = curl_multi_setopt(multi, + CURLMOPT_SOCKETFUNCTION, curl_socket)) != CURLM_OK) + fatal("curl_multi_setopt: %s", curl_multi_strerror(res)); + + if ((res = curl_multi_setopt(multi, + CURLMOPT_TIMERFUNCTION, curl_timer)) != CURLM_OK) + fatal("curl_multi_setopt: %s", curl_multi_strerror(res)); + + for (i = 0; i < FD_CACHE_BUCKETS; i++) + LIST_INIT(&cache[i]); + + kore_pool_init(&fd_cache_pool, "fd_cache_pool", 100, + sizeof(struct fd_cache)); +} + +int +kore_curl_init(struct kore_curl *client, const char *url) +{ + CURL *handle; + + memset(client, 0, sizeof(*client)); + + TAILQ_INIT(&client->http.resp_hdrs); + + if ((handle = curl_easy_init()) == NULL) + return (KORE_RESULT_ERROR); + + curl_easy_setopt(handle, CURLOPT_WRITEDATA, &client->response); + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, kore_curl_tobuf); + + curl_easy_setopt(handle, CURLOPT_URL, url); + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(handle, CURLOPT_PRIVATE, client); + curl_easy_setopt(handle, CURLOPT_TIMEOUT, kore_curl_timeout); + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, client->errbuf); + + client->handle = handle; + client->url = kore_strdup(url); + client->type = KORE_CURL_TYPE_CUSTOM; + + return (KORE_RESULT_OK); +} + +void +kore_curl_cleanup(struct kore_curl *client) +{ + struct http_header *hdr, *next; + + kore_free(client->url); + + if (client->flags & KORE_CURL_FLAG_BOUND) + LIST_REMOVE(client, list); + + if (client->handle != NULL) { + curl_multi_remove_handle(multi, client->handle); + curl_easy_cleanup(client->handle); + } + + if (client->http.hdrlist != NULL) + curl_slist_free_all(client->http.hdrlist); + + if (client->response != NULL) + kore_buf_free(client->response); + + if (client->http.headers != NULL) + kore_buf_free(client->http.headers); + + if (client->http.tosend != NULL) + kore_buf_free(client->http.tosend); + + for (hdr = TAILQ_FIRST(&client->http.resp_hdrs); + hdr != NULL; hdr = next) { + next = TAILQ_NEXT(hdr, list); + TAILQ_REMOVE(&client->http.resp_hdrs, hdr, list); + kore_pool_put(&http_header_pool, hdr); + } +} + +size_t +kore_curl_tobuf(char *ptr, size_t size, size_t nmemb, void *udata) +{ + size_t len; + struct kore_buf **buf, *b; + + if (SIZE_MAX / nmemb < size) + fatal("%s: %zu * %zu overflow", __func__, nmemb, size); + + buf = udata; + len = size * nmemb; + + if (*buf == NULL) + *buf = kore_buf_alloc(len); + + b = *buf; + + if (b->offset + len < b->offset) + fatal("%s: %zu+%zu overflows", __func__, b->offset, len); + + if ((b->offset + len) > kore_curl_recv_max) { + kore_log(LOG_ERR, + "received too large transfer (%zu > %" PRIu64 ")", + b->offset + len, kore_curl_recv_max); + return (0); + } + + kore_buf_append(b, ptr, len); + + return (len); +} + +size_t +kore_curl_frombuf(char *ptr, size_t size, size_t nmemb, void *udata) +{ + size_t len; + struct kore_buf *buf; + + if (SIZE_MAX / nmemb < size) + fatal("%s: %zu * %zu overflow", __func__, nmemb, size); + + buf = udata; + len = size * nmemb; + + if (buf->offset == buf->length) + return (0); + + if (buf->offset + len < buf->offset) + fatal("%s: %zu+%zu overflows", __func__, buf->offset, len); + + if ((buf->offset + len) < buf->length) { + memcpy(ptr, buf->data + buf->offset, len); + } else { + len = buf->length - buf->offset; + memcpy(ptr, buf->data + buf->offset, len); + } + + buf->offset += len; + + return (len); +} + +void +kore_curl_bind_request(struct kore_curl *client, struct http_request *req) +{ + if (client->cb != NULL) + fatal("%s: already bound to callback", __func__); + + client->req = req; + http_request_sleep(req); + + client->flags |= KORE_CURL_FLAG_BOUND; + LIST_INSERT_HEAD(&req->chandles, client, list); +} + +void +kore_curl_bind_callback(struct kore_curl *client, + void (*cb)(struct kore_curl *, void *), void *arg) +{ + if (client->req != NULL) + fatal("%s: already bound to request", __func__); + + client->cb = cb; + client->arg = arg; +} + +void +kore_curl_run(struct kore_curl *client) +{ + curl_multi_add_handle(multi, client->handle); +} + +int +kore_curl_success(struct kore_curl *client) +{ + return (client->result == CURLE_OK); +} + +void +kore_curl_logerror(struct kore_curl *client) +{ + kore_log(LOG_NOTICE, "curl error: %s -> %s", + client->url, client->errbuf); +} + +void +kore_curl_response_as_bytes(struct kore_curl *client, const u_int8_t **body, + size_t *len) +{ + if (client->response == NULL) { + *len = 0; + *body = NULL; + } else { + *len = client->response->offset; + *body = client->response->data; + } +} + +char * +kore_curl_response_as_string(struct kore_curl *client) +{ + kore_buf_stringify(client->response, NULL); + + return ((char *)client->response->data); +} + +void +kore_curl_http_setup(struct kore_curl *client, int method, const void *data, + size_t len) +{ + const char *mname; + int has_body; + + if (client->handle == NULL) + fatal("%s: called without setup", __func__); + + mname = NULL; + has_body = 1; + + client->type = KORE_CURL_TYPE_HTTP_CLIENT; + + curl_easy_setopt(client->handle, CURLOPT_HEADERDATA, + &client->http.headers); + curl_easy_setopt(client->handle, CURLOPT_HEADERFUNCTION, + kore_curl_tobuf); + + kore_curl_http_set_header(client, "expect", ""); + + switch (method) { + case HTTP_METHOD_GET: + break; + case HTTP_METHOD_HEAD: + curl_easy_setopt(client->handle, CURLOPT_NOBODY, 1); + break; + case HTTP_METHOD_DELETE: + case HTTP_METHOD_OPTIONS: + break; + case HTTP_METHOD_PUT: + has_body = 1; + curl_easy_setopt(client->handle, CURLOPT_UPLOAD, 1); + break; + case HTTP_METHOD_PATCH: + mname = http_method_text(method); + /* fallthrough */ + case HTTP_METHOD_POST: + has_body = 1; + curl_easy_setopt(client->handle, CURLOPT_POST, 1); + break; + default: + fatal("%s: unknown method %d", __func__, method); + } + + if (has_body && data != NULL && len > 0) { + client->http.tosend = kore_buf_alloc(len); + kore_buf_append(client->http.tosend, data, len); + kore_buf_reset(client->http.tosend); + + curl_easy_setopt(client->handle, CURLOPT_READDATA, + client->http.tosend); + curl_easy_setopt(client->handle, CURLOPT_READFUNCTION, + kore_curl_frombuf); + } + + if (has_body) { + if (method == HTTP_METHOD_PUT) { + curl_easy_setopt(client->handle, + CURLOPT_INFILESIZE_LARGE, len); + } else { + curl_easy_setopt(client->handle, + CURLOPT_POSTFIELDSIZE_LARGE, len); + } + } else { + if (data != NULL || len != 0) { + fatal("%s: %d should not have a body", + __func__, method); + } + } + + if (mname != NULL) + curl_easy_setopt(client->handle, CURLOPT_CUSTOMREQUEST, mname); +} + +void +kore_curl_http_set_header(struct kore_curl *client, const char *header, + const char *value) +{ + struct kore_buf buf; + const char *hdr; + + kore_buf_init(&buf, 512); + + if (value != NULL || *value != '\0') { + kore_buf_appendf(&buf, "%s: %s", header, value); + } else { + kore_buf_appendf(&buf, "%s:", header); + } + + hdr = kore_buf_stringify(&buf, NULL); + + client->http.hdrlist = curl_slist_append(client->http.hdrlist, hdr); + kore_buf_cleanup(&buf); + + curl_easy_setopt(client->handle, + CURLOPT_HTTPHEADER, client->http.hdrlist); +} + +int +kore_curl_http_get_header(struct kore_curl *client, const char *header, + const char **out) +{ + struct http_header *hdr; + + if (!(client->flags & KORE_CURL_FLAG_HTTP_PARSED_HEADERS)) + kore_curl_http_parse_headers(client); + + TAILQ_FOREACH(hdr, &(client->http.resp_hdrs), list) { + if (!strcasecmp(hdr->header, header)) { + *out = hdr->value; + return (KORE_RESULT_OK); + } + } + + return (KORE_RESULT_ERROR); +} + +void +kore_curl_http_parse_headers(struct kore_curl *client) +{ + struct http_header *hdr; + int i, cnt; + char *value, *hbuf, *headers[HTTP_REQ_HEADER_MAX]; + + if (client->flags & KORE_CURL_FLAG_HTTP_PARSED_HEADERS) + fatal("%s: headers already parsed", __func__); + + client->flags |= KORE_CURL_FLAG_HTTP_PARSED_HEADERS; + + if (client->http.headers == NULL) + return; + + hbuf = kore_buf_stringify(client->http.headers, NULL); + cnt = kore_split_string(hbuf, "\r\n", headers, HTTP_REQ_HEADER_MAX); + + for (i = 0; i < cnt; i++) { + if ((value = http_validate_header(headers[i])) == NULL) + continue; + + if (*value == '\0') + continue; + + hdr = kore_pool_get(&http_header_pool); + hdr->header = headers[i]; + hdr->value = value; + TAILQ_INSERT_TAIL(&(client->http.resp_hdrs), hdr, list); + } +} + +static int +curl_socket(CURL *easy, curl_socket_t fd, int action, void *arg, void *sock) +{ + CURLcode res; + struct fd_cache *fdc; + struct kore_curl *client; + + client = NULL; + + res = curl_easy_getinfo(easy, CURLINFO_PRIVATE, &client); + if (res != CURLE_OK) + fatal("curl_easy_getinfo: %s", curl_easy_strerror(res)); + + if (client == NULL) + fatal("%s: failed to get client context", __func__); + + fdc = fd_cache_get(fd); + + switch (action) { + case CURL_POLL_NONE: + break; + case CURL_POLL_IN: + case CURL_POLL_OUT: + case CURL_POLL_INOUT: + if (fdc->scheduled == 0) { + kore_platform_event_all(fd, fdc); + fdc->scheduled = 1; + } + break; + case CURL_POLL_REMOVE: + if (fdc->scheduled) { + fdc->evt.flags = 0; + fdc->scheduled = 0; + kore_platform_disable_read(fd); +#if !defined(__linux__) + kore_platform_disable_write(fd); +#endif + } + break; + default: + fatal("unknown action value: %d", action); + } + + /* + * XXX - libcurl hates edge triggered io. + */ + if (action == CURL_POLL_OUT || action == CURL_POLL_INOUT) { + if (fdc->evt.flags & KORE_EVENT_WRITE) { + if (fdc->scheduled) { + kore_platform_disable_read(fdc->fd); +#if !defined(__linux__) + kore_platform_disable_write(fdc->fd); +#endif + } + + fdc->evt.flags = 0; + kore_platform_event_all(fdc->fd, fdc); + } + } + + return (CURLM_OK); +} + +static void +curl_process(void) +{ + CURLcode res; + CURLMsg *msg; + CURL *handle; + struct kore_curl *client; + int pending; + + pending = 0; + + while ((msg = curl_multi_info_read(multi, &pending)) != NULL) { + if (msg->msg != CURLMSG_DONE) + continue; + + handle = msg->easy_handle; + + res = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &client); + if (res != CURLE_OK) + fatal("curl_easy_getinfo: %s", curl_easy_strerror(res)); + + if (client == NULL) + fatal("%s: failed to get client context", __func__); + + client->result = msg->data.result; + + if (client->type == KORE_CURL_TYPE_HTTP_CLIENT) { + curl_easy_getinfo(client->handle, + CURLINFO_RESPONSE_CODE, &client->http.status); + } + + curl_multi_remove_handle(multi, client->handle); + curl_easy_cleanup(client->handle); + + client->handle = NULL; + + if (client->req != NULL) + http_request_wakeup(client->req); + else if (client->cb != NULL) + client->cb(client, client->arg); + } +} + +static void +curl_timeout(void *uarg, u_int64_t now) +{ + CURLMcode res; + + timer = NULL; + + res = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &running); + if (res != CURLM_OK) + fatal("curl_multi_socket_action: %s", curl_multi_strerror(res)); + + curl_process(); +} + +static int +curl_timer(CURLM *mctx, long timeout, void *arg) +{ + if (timeout < 0) { + if (timer != NULL) { + kore_timer_remove(timer); + timer = NULL; + } + return (CURLM_OK); + } + + if (timer != NULL) { + kore_timer_remove(timer); + timer = NULL; + } + + if (timeout == 0) + timeout = 10; + + timer = kore_timer_add(curl_timeout, timeout, mctx, KORE_TIMER_ONESHOT); + + return (CURLM_OK); +} + +static void +curl_event_handle(void *arg, int eof) +{ + int flags; + ssize_t bytes; + char buf[32]; + struct fd_cache *fdc = arg; + + flags = 0; + + if (fdc->evt.flags & KORE_EVENT_READ) + flags |= CURL_CSELECT_IN; + + if (fdc->evt.flags & KORE_EVENT_WRITE) + flags |= CURL_CSELECT_OUT; + + if (eof) + flags = CURL_CSELECT_ERR; + + curl_multi_socket_action(multi, fdc->fd, flags, &running); + + /* + * XXX - libcurl doesn't work with edge triggered i/o so check + * if we need to reprime the event. Not optimal. + */ + if (fdc->evt.flags & KORE_EVENT_READ) { + bytes = recv(fdc->fd, buf, sizeof(buf), MSG_PEEK); + if (bytes > 0) { + if (fdc->scheduled) { + kore_platform_disable_read(fdc->fd); +#if !defined(__linux__) + kore_platform_disable_write(fdc->fd); +#endif + } + + fdc->evt.flags = 0; + kore_platform_event_all(fdc->fd, fdc); + } + } + + curl_process(); +} + +static struct fd_cache * +fd_cache_get(int fd) +{ + struct fd_cache *fdc; + int bucket; + + bucket = fd % FD_CACHE_BUCKETS; + + LIST_FOREACH(fdc, &cache[bucket], list) { + if (fdc->fd == fd) + return (fdc); + } + + fdc = kore_pool_get(&fd_cache_pool); + + fdc->fd = fd; + fdc->scheduled = 0; + + fdc->evt.flags = 0; + fdc->evt.handle = curl_event_handle; + fdc->evt.type = KORE_TYPE_CURL_HANDLE; + + LIST_INSERT_HEAD(&cache[bucket], fdc, list); + + return (fdc); +} diff --git a/src/http.c b/src/http.c index 92040df..103ef4e 100644 --- a/src/http.c +++ b/src/http.c @@ -43,6 +43,10 @@ #include "tasks.h" #endif +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + static struct { const char *ext; const char *type; @@ -131,7 +135,6 @@ static int multipart_parse_headers(struct http_request *, struct kore_buf *, struct kore_buf *, const char *, const int); -static char *http_validate_header(char *); static struct http_request *http_request_new(struct connection *, const char *, const char *, char *, const char *); @@ -144,10 +147,11 @@ static TAILQ_HEAD(, http_request) http_requests; static TAILQ_HEAD(, http_request) http_requests_sleeping; static LIST_HEAD(, http_media_type) http_media_types; static struct kore_pool http_request_pool; -static struct kore_pool http_header_pool; static struct kore_pool http_cookie_pool; static struct kore_pool http_body_path; +struct kore_pool http_header_pool; + u_int32_t http_request_count = 0; u_int32_t http_request_ms = HTTP_REQUEST_MS; u_int16_t http_body_timeout = HTTP_BODY_TIMEOUT; @@ -393,6 +397,9 @@ http_request_free(struct http_request *req) #endif #if defined(KORE_USE_PGSQL) struct kore_pgsql *pgsql; +#endif +#if defined(KORE_USE_CURL) + struct kore_curl *client; #endif struct http_file *f, *fnext; struct http_arg *q, *qnext; @@ -428,7 +435,12 @@ http_request_free(struct http_request *req) kore_pgsql_cleanup(pgsql); } #endif - +#if defined(KORE_USE_CURL) + while (!LIST_EMPTY(&req->chandles)) { + client = LIST_FIRST(&req->chandles); + kore_curl_cleanup(client); + } +#endif kore_debug("http_request_free: %p->%p", req->owner, req); kore_free(req->headers); @@ -2208,7 +2220,7 @@ http_media_type(const char *path) return (NULL); } -static char * +char * http_validate_header(char *header) { u_int8_t idx; diff --git a/src/kore.c b/src/kore.c index 23928e3..4025cb5 100644 --- a/src/kore.c +++ b/src/kore.c @@ -33,6 +33,10 @@ #include "http.h" #endif +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + #if defined(KORE_USE_PYTHON) #include "python_api.h" #endif @@ -105,6 +109,9 @@ version(void) #if defined(KORE_NO_HTTP) printf("no-http "); #endif +#if defined(KORE_USE_CURL) + printf("curl "); +#endif #if defined(KORE_USE_PGSQL) printf("pgsql "); #endif @@ -192,6 +199,9 @@ main(int argc, char *argv[]) kore_log_init(); #if !defined(KORE_NO_HTTP) http_parent_init(); +#if defined(KORE_USE_CURL) + kore_curl_sysinit(); +#endif kore_auth_init(); kore_validator_init(); kore_filemap_init(); diff --git a/src/python.c b/src/python.c index 577901b..3d3d44c 100644 --- a/src/python.c +++ b/src/python.c @@ -33,6 +33,10 @@ #include "pgsql.h" #endif +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + #include "python_api.h" #include "python_methods.h" @@ -77,6 +81,12 @@ static PyObject *pykore_pgsql_alloc(struct http_request *, const char *, const char *); #endif +#if defined(KORE_USE_CURL) +static void python_curl_callback(struct kore_curl *, void *); +static PyObject *pyhttp_client_request(struct pyhttp_client *, int, + PyObject *); +#endif + static void python_append_path(const char *); static void python_push_integer(PyObject *, const char *, long); static void python_push_type(const char *, PyObject *, PyTypeObject *); @@ -963,6 +973,10 @@ python_module_init(void) python_push_type("pysocket", pykore, &pysocket_type); python_push_type("pyconnection", pykore, &pyconnection_type); +#if defined(KORE_USE_CURL) + python_push_type("pyhttpclient", pykore, &pyhttp_client_type); +#endif + python_push_type("pyhttp_file", pykore, &pyhttp_file_type); python_push_type("pyhttp_request", pykore, &pyhttp_request_type); @@ -2337,6 +2351,7 @@ pysocket_async_recv(struct pysocket_op *op) return (NULL); } + Py_DECREF(tuple); PyErr_SetObject(PyExc_StopIteration, result); Py_DECREF(result); @@ -3743,3 +3758,323 @@ pyhttp_pgsql(struct pyhttp_request *pyreq, PyObject *args) return ((PyObject *)obj); } #endif + +#if defined(KORE_USE_CURL) +static PyObject * +python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *obj; + struct pyhttp_client *client; + const char *url, *v; + + if (!PyArg_ParseTuple(args, "s", &url)) + return (NULL); + + client = PyObject_New(struct pyhttp_client, &pyhttp_client_type); + if (client == NULL) + return (NULL); + + client->tlskey = NULL; + client->tlscert = NULL; + client->url = kore_strdup(url); + + if (kwargs != NULL) { + if ((obj = PyDict_GetItemString(kwargs, "tlscert")) != NULL) { + if ((v = PyUnicode_AsUTF8(obj)) == NULL) { + Py_DECREF((PyObject *)client); + return (NULL); + } + + client->tlscert = kore_strdup(v); + } + + if ((obj = PyDict_GetItemString(kwargs, "tlskey")) != NULL) { + if ((v = PyUnicode_AsUTF8(obj)) == NULL) { + Py_DECREF((PyObject *)client); + return (NULL); + } + + client->tlskey = kore_strdup(v); + } + } + + if ((client->tlscert != NULL && client->tlskey == NULL) || + (client->tlskey != NULL && client->tlscert == NULL)) { + Py_DECREF((PyObject *)client); + PyErr_SetString(PyExc_RuntimeError, + "invalid TLS client configuration"); + return (NULL); + } + + return ((PyObject *)client); +} + +static void +pyhttp_client_dealloc(struct pyhttp_client *client) +{ + kore_free(client->url); + kore_free(client->tlskey); + kore_free(client->tlscert); + + PyObject_Del((PyObject *)client); +} + +static PyObject * +pyhttp_client_get(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_GET, kwargs)); +} + +static PyObject * +pyhttp_client_put(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_PUT, kwargs)); +} + +static PyObject * +pyhttp_client_post(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_POST, kwargs)); +} + +static PyObject * +pyhttp_client_head(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_HEAD, kwargs)); +} + +static PyObject * +pyhttp_client_patch(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_PATCH, kwargs)); +} + +static PyObject * +pyhttp_client_delete(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_DELETE, kwargs)); +} + +static PyObject * +pyhttp_client_options(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_OPTIONS, kwargs)); +} + +static PyObject * +pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) +{ + struct pyhttp_client_op *op; + char *ptr; + const char *k, *v; + Py_ssize_t length, idx; + PyObject *data, *headers, *key, *item; + + ptr = NULL; + length = 0; + headers = NULL; + + if (kwargs != NULL && + ((headers = PyDict_GetItemString(kwargs, "headers")) != NULL)) { + if (!PyDict_CheckExact(headers)) { + PyErr_SetString(PyExc_RuntimeError, + "header keyword must be a dict"); + return (NULL); + } + } + + switch (m) { + case HTTP_METHOD_GET: + case HTTP_METHOD_HEAD: + case HTTP_METHOD_DELETE: + case HTTP_METHOD_OPTIONS: + break; + case HTTP_METHOD_PUT: + case HTTP_METHOD_POST: + case HTTP_METHOD_PATCH: + length = -1; + + if (kwargs == NULL) { + PyErr_Format(PyExc_RuntimeError, + "no keyword arguments given, but body expected ", + http_method_text(m)); + return (NULL); + } + + if ((data = PyDict_GetItemString(kwargs, "body")) == NULL) + return (NULL); + + if (PyBytes_AsStringAndSize(data, &ptr, &length) == -1) + return (NULL); + + if (length < 0) { + PyErr_SetString(PyExc_TypeError, "invalid length"); + return (NULL); + } + break; + default: + fatal("%s: unknown method %d", __func__, m); + } + + op = PyObject_New(struct pyhttp_client_op, &pyhttp_client_op_type); + if (op == NULL) + return (NULL); + + if (!kore_curl_init(&op->curl, client->url)) { + Py_DECREF((PyObject *)op); + PyErr_SetString(PyExc_RuntimeError, "failed to setup call"); + return (NULL); + } + + op->headers = 0; + op->coro = coro_running; + op->state = PYHTTP_CLIENT_OP_RUN; + + Py_INCREF(client); + + kore_curl_http_setup(&op->curl, m, ptr, length); + kore_curl_bind_callback(&op->curl, python_curl_callback, op); + + /* Go in with our own bare hands. */ + if (client->tlskey != NULL && client->tlscert != NULL) { + curl_easy_setopt(op->curl.handle, CURLOPT_SSLCERT, + client->tlscert); + curl_easy_setopt(op->curl.handle, CURLOPT_SSLKEY, + client->tlskey); + } + + if (headers != NULL) { + idx = 0; + while (PyDict_Next(headers, &idx, &key, &item)) { + if ((k = PyUnicode_AsUTF8(key)) == NULL) { + Py_DECREF((PyObject *)op); + return (NULL); + } + + if ((v = PyUnicode_AsUTF8(item)) == NULL) { + Py_DECREF((PyObject *)op); + return (NULL); + } + + kore_curl_http_set_header(&op->curl, k, v); + } + } + + if (kwargs != NULL && + ((item = PyDict_GetItemString(kwargs, "return_headers")) != NULL)) { + if (item == Py_True) { + op->headers = 1; + } else if (item == Py_False) { + op->headers = 0; + } else { + Py_DECREF((PyObject *)op); + PyErr_SetString(PyExc_RuntimeError, + "return_headers not True or False"); + return (NULL); + } + } + + return ((PyObject *)op); +} + +static void +pyhttp_client_op_dealloc(struct pyhttp_client_op *op) +{ + kore_curl_cleanup(&op->curl); + PyObject_Del((PyObject *)op); +} + +static PyObject * +pyhttp_client_op_await(PyObject *op) +{ + Py_INCREF(op); + return (op); +} + +static PyObject * +pyhttp_client_op_iternext(struct pyhttp_client_op *op) +{ + size_t len; + struct http_header *hdr; + const u_int8_t *response; + PyObject *result, *tuple, *dict, *value; + + if (op->state == PYHTTP_CLIENT_OP_RUN) { + kore_curl_run(&op->curl); + op->state = PYHTTP_CLIENT_OP_RESULT; + Py_RETURN_NONE; + } + + if (!kore_curl_success(&op->curl)) { + PyErr_Format(PyExc_RuntimeError, "request to '%s' failed: %s", + op->curl.url, op->curl.errbuf); + return (NULL); + } + + kore_curl_response_as_bytes(&op->curl, &response, &len); + + if (op->headers) { + kore_curl_http_parse_headers(&op->curl); + + if ((dict = PyDict_New()) == NULL) + return (NULL); + + TAILQ_FOREACH(hdr, &op->curl.http.resp_hdrs, list) { + value = PyUnicode_FromString(hdr->value); + if (value == NULL) { + Py_DECREF(dict); + return (NULL); + } + + if (PyDict_SetItemString(dict, + hdr->header, value) == -1) { + Py_DECREF(dict); + Py_DECREF(value); + return (NULL); + } + + Py_DECREF(value); + } + + if ((tuple = Py_BuildValue("(iOy#)", op->curl.http.status, + dict, (const char *)response, len)) == NULL) + return (NULL); + + Py_DECREF(dict); + } else { + if ((tuple = Py_BuildValue("(iy#)", op->curl.http.status, + (const char *)response, len)) == NULL) + return (NULL); + } + + result = PyObject_CallFunctionObjArgs(PyExc_StopIteration, tuple, NULL); + if (result == NULL) { + Py_DECREF(tuple); + return (NULL); + } + + Py_DECREF(tuple); + PyErr_SetObject(PyExc_StopIteration, result); + Py_DECREF(result); + + return (NULL); +} + +static void +python_curl_callback(struct kore_curl *curl, void *arg) +{ + struct pyhttp_client_op *op = arg; + + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); +} +#endif diff --git a/src/utils.c b/src/utils.c index 489edae..d7a6e4d 100644 --- a/src/utils.c +++ b/src/utils.c @@ -639,7 +639,7 @@ static void fatal_log(const char *fmt, va_list args) { char buf[2048]; - extern const char *__progname; + extern const char *kore_progname; (void)vsnprintf(buf, sizeof(buf), fmt, args); @@ -651,5 +651,5 @@ fatal_log(const char *fmt, va_list args) kore_keymgr_cleanup(1); #endif - printf("%s: %s\n", __progname, buf); + printf("%s: %s\n", kore_progname, buf); }