diff --git a/examples/python-async/src/async_process.py b/examples/python-async/src/async_process.py index 39fed21..2add851 100644 --- a/examples/python-async/src/async_process.py +++ b/examples/python-async/src/async_process.py @@ -41,9 +41,16 @@ async def async_proc(req): # Read until EOF (None is returned) while True: - chunk = await proc.recv(1024) - if chunk is None: - break + try: + # Read from the process, with an optional 1 second timeout. + # The recv() call will throw a TimeoutError exception if + # the timeout has elapsed before any data was read. + chunk = await proc.recv(1024, 1000) + if chunk is None: + break + except TimeoutError as e: + print("recv() timed out: %s" % e) + continue stdout += chunk.decode() # Reap the process. diff --git a/examples/python-echo/src/echo.py b/examples/python-echo/src/echo.py index 81ebc1e..ea42b24 100644 --- a/examples/python-echo/src/echo.py +++ b/examples/python-echo/src/echo.py @@ -22,6 +22,7 @@ class EchoServer: def __init__(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setblocking(False) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(("127.0.0.1", 6969)) sock.listen() @@ -40,13 +41,21 @@ class EchoServer: kore.fatal("exception %s" % e) # Each client will run as this co-routine. + # In this case we pass a timeout of 1 second to the recv() call + # which will throw a TimeoutError exception in case the timeout + # is hit before data is read from the socket. + # + # This timeout argument is optional. If none is specified the call + # will wait until data becomes available. async def handle_client(self, client): while True: try: - data = await client.recv(1024) + data = await client.recv(1024, 1000) if data is None: break await client.send(data) + except TimeoutError as e: + print("timed out reading (%s)" % e) except Exception as e: print("client got exception %s" % e) client.close() diff --git a/examples/python/.gitignore b/examples/python/.gitignore deleted file mode 100644 index ac72328..0000000 --- a/examples/python/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.o -.flavor -.objs -python.so -assets.h -cert diff --git a/examples/python/README.md b/examples/python/README.md deleted file mode 100644 index bb67224..0000000 --- a/examples/python/README.md +++ /dev/null @@ -1,18 +0,0 @@ -Kore python module example. - -This application requires kore to be built with PYTHON=1. - -It mixes native code (dso) with python code. - -Run: -``` - $ kodev run -``` - -Test: -``` - $ curl -k https://127.0.0.1:8888 - $ curl -k https://127.0.0.1:8888/state - $ curl -k https://127.0.0.1:8888/auth - $ curl -X PUT -d '{\"hello\": 123}' https://127.0.0.1:8888/json -``` diff --git a/examples/python/assets/frontend.html b/examples/python/assets/frontend.html deleted file mode 100644 index 700c821..0000000 --- a/examples/python/assets/frontend.html +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - -
- -
- -
-
- - - diff --git a/examples/python/conf/build.conf b/examples/python/conf/build.conf deleted file mode 100644 index 55b3465..0000000 --- a/examples/python/conf/build.conf +++ /dev/null @@ -1,35 +0,0 @@ -# python build config -# You can switch flavors using: kodev flavor [newflavor] - -# Set to yes if you wish to produce a single binary instead -# of a dynamic library. If you set this to yes you must also -# set kore_source together with kore_flavor and update ldflags -# to include the appropriate libraries you will be linking with. -#single_binary=no -#kore_source=/home/joris/src/kore -#kore_flavor= - -# The flags below are shared between flavors -cflags=-Wall -Wmissing-declarations -Wshadow -cflags=-Wstrict-prototypes -Wmissing-prototypes -cflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -cxxflags=-Wall -Wmissing-declarations -Wshadow -cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -# Mime types for assets served via the builtin asset_serve_* -#mime_add=txt:text/plain; charset=utf-8 -#mime_add=png:image/png -mime_add=html:text/html; charset=utf-8 - -dev { - # These flags are added to the shared ones when - # you build the "dev" flavor. - cflags=-g - cxxflags=-g -} - -#prod { -# You can specify additional flags here which are only -# included if you build with the "prod" flavor. -#} diff --git a/examples/python/conf/python.conf b/examples/python/conf/python.conf deleted file mode 100644 index 76c6ef9..0000000 --- a/examples/python/conf/python.conf +++ /dev/null @@ -1,57 +0,0 @@ -# python configuration - -load ./python.so onload - -# import both python modules. -python_import src/index.py onload -python_import src/websockets.py -python_import src/upload.py - -bind 127.0.0.1 8888 - -tls_dhparam dh2048.pem - -validator v_id function c_validator -validator v_p_id function python_validator -validator v_auth function python_auth - -websocket_maxframe 65536 -websocket_timeout 20 - -authentication auth { - authentication_type request - authentication_validator v_auth -} - -domain * { - certfile cert/server.pem - certkey cert/key.pem - - # Mix page handlers between native and python. - static / page - static /c cpage - static /b minimal - static /json json_parse - static /ws ws_connect - static /upload upload - static /kaka kaka - - # Use the builtin asset_serve_* to serve frontend HTML. - static /wspage asset_serve_frontend_html - - static /auth page auth - - # - # On the native page handler, use a python validator. - # - params qs:get /c { - validate id v_p_id - } - - # - # On the python page handler, use a native validator. - # - params qs:get / { - validate id v_id - } -} diff --git a/examples/python/src/index.py b/examples/python/src/index.py deleted file mode 100644 index c05b252..0000000 --- a/examples/python/src/index.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Copyright (c) 2017-2018 Joris Vink -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -# This is a simple python module that can be loaded into Kore. -# It demonstrates some basic abilities to deal with HTTP requests. - -# Pull in the kore stuff. -import kore - -# Pull in python JSON parsing. -import json - -# -# A validator that the configuration for this application uses to determine -# if a request fulfills the requirements to pass an authentication block. -# -# See the configuration for more. -# -def python_auth(req, data): - kore.log(kore.LOG_NOTICE, "python auth called %s" % data) - return kore.RESULT_OK - -# -# Define a validator that kore can use via the configuration to validate -# something before allowing access to it. -# -def python_validator(req, data): - kore.log(kore.LOG_NOTICE, "python validator called %s" % data) - return kore.RESULT_OK - -# -# This function is called when our python module is loaded/unloaded. -# The action param is kore.MODULE_LOAD or kore.MODULE_UNLOAD respectively. -# -def onload(action): - kore.log(kore.LOG_INFO, "python module onload called with %d!" % action) - return kore.RESULT_OK - -# Called by Kore when the parent is starting. -def kore_parent_configure(): - # Listen on an additional interface and port. - kore.listen("127.0.0.1", "8889", "") - kore.log(kore.LOG_INFO, "kore_parent_configure called!") - -# Called by Kore when the worker is starting. -def kore_worker_configure(): - kore.log(kore.LOG_INFO, "kore_worker_configure called!") - -# -# Test page handler that displays some debug information as well as -# fetches the "xframe" header from the request and logs it if present. -# -# If the request is a POST then we read the body up to 1024 bytes in -# one go and display the result and bytes read in the log. -# -# If it's a GET request attempts to find the "id" argument and presents -# it to the user. -# -def page(req): - kore.log(kore.LOG_INFO, - "%s path is %s - host is %s" % (req, req.path, req.host)) - kore.log(kore.LOG_INFO, "connection is %s" % req.connection) - xframe = req.request_header("xframe") - if xframe != None: - kore.log(kore.LOG_INFO, "xframe header present: '%s'" % xframe) - if req.method == kore.HTTP_METHOD_POST: - try: - length, body = req.body_read(1024) - kore.log(kore.LOG_INFO, "POST and got %d bytes! (%s)" % - (length, body.decode("utf-8"))) - except RuntimeError as r: - kore.log(kore.LOG_INFO, "oops runtime error %s" % r) - req.response(500, b'') - except: - kore.log(kore.LOG_INFO, "oops other error") - req.response(500, b'') - else: - req.response_header("content-type", "text/plain") - req.response(200, body) - else: - req.populate_get() - id = req.argument("id") - if id != None: - kore.log(kore.LOG_INFO, "got id of %s" % id) - req.response_header("content-type", "text/plain") - req.response(200, "hello 1234".encode("utf-8")) - -# -# Handler that parses the incoming body as JSON and dumps out some things. -# -def json_parse(req): - if req.method != kore.HTTP_METHOD_PUT: - req.response(400, b'') - else: - data = json.loads(req.body) - kore.log(kore.LOG_INFO, "loaded json %s" % data) - if data["hello"] == 123: - kore.log(kore.LOG_INFO, "hello is 123!") - - req.response(200, "ok".encode("utf-8")) - -# -# Small handler, returns 200 OK. -# -def minimal(req): - req.response(200, b'') - -# -# Small handler that grabs a cookie if set. -# -def kaka(req): - req.populate_cookies() - cookie = req.cookie("hello") - if cookie is not None: - kore.log(kore.LOG_INFO, "got hello with value %s" % cookie) - req.response(200, b'') diff --git a/examples/python/src/python.c b/examples/python/src/python.c deleted file mode 100644 index 9499b3e..0000000 --- a/examples/python/src/python.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2017-2018 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include -#include - -/* - * Just some examples of things that can be mixed with python modules. - */ - -int onload(int); -int cpage(struct http_request *); -int c_validator(struct http_request *, void *); - -int -c_validator(struct http_request *req, void *data) -{ - kore_log(LOG_NOTICE, "c_validator(): called!"); - return (KORE_RESULT_OK); -} - -int -onload(int action) -{ - kore_log(LOG_NOTICE, "onload called from native"); - return (KORE_RESULT_OK); -} - -int -cpage(struct http_request *req) -{ - http_populate_get(req); - http_response(req, 200, "native", 6); - - return (KORE_RESULT_OK); -} diff --git a/examples/python/src/upload.py b/examples/python/src/upload.py deleted file mode 100644 index e0e9dda..0000000 --- a/examples/python/src/upload.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) 2017-2018 Joris Vink -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -# Processing incoming files in a multipart form. - -import kore - -# -# This handler receives a POST with a multipart data. -# It extracts the file called "file" and writes it to a new file. -# -def upload(req): - # We only allow POST's. - if req.method is not kore.HTTP_METHOD_POST: - req.response_header("allow", "post") - req.response(400, b'') - return - - # Ask kore to parse incoming multipart data. - req.populate_multi() - - # Lookup the file called "file". - file = req.file_lookup("file") - if not file: - req.response(400, b'') - return - - kore.log(kore.LOG_INFO, - "%s (%s, filename=%s)" % (file, file.name, file.filename)) - - # Open target file. - f = open(file.filename, "wb") - if not f: - req.response(500, b'') - return - - # Read all data from incoming file and write it to the output file. - len = True - while len: - len, bytes = file.read(1024) - kore.log(kore.LOG_INFO, "got %d bytes of data" % len) - f.write(bytes) - - f.close() - req.response(200, b'') diff --git a/examples/python/src/websockets.py b/examples/python/src/websockets.py deleted file mode 100644 index dc950ad..0000000 --- a/examples/python/src/websockets.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2018 Joris Vink -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -# Using kore websockets via python. - -import kore - -# -# Our connection callback, gets called for each new websocket connection. -# -def onconnect(c): - kore.log(kore.LOG_INFO, "%s: py connected" % c) - -# -# Each websocket arriving on a connection triggers this function. -# -# It receives the connection object, the opcode (TEXT/BINARY) and the -# actual data received. -# -# In this example we use the websocket_broadcast() method from kore to -# simply relay the message to all other connection clients. -# -# If you want to send data directly back to the connection you can -# use kore.websocket_send(connection, op, data) -# -def onmessage(c, op, data): - kore.websocket_broadcast(c, op, data, kore.WEBSOCKET_BROADCAST_GLOBAL) - #c.websocket_send(op, data) - -# -# Called for every connection that goes byebye. -# -def ondisconnect(c): - kore.log(kore.LOG_INFO, "%s: py disconnecting" % c) - -# -# The /ws connection handler. It establishes the websocket connection -# after a request was made for it. -# -# Note that the websocket_handshake() method for the request takes 3 -# parameters which are the connection callback, message callback and -# disconnect callback. -# -# These are given as strings to Kore which will then resolve them -# in all modules which means you can give native callbacks here as well. -# -def ws_connect(req): - try: - req.websocket_handshake("onconnect", "onmessage", "ondisconnect") - except: - req.response(500, b'') diff --git a/include/kore/kore.h b/include/kore/kore.h index 79f9141..850ec85 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -22,6 +22,7 @@ #endif #include +#include #include #include #include diff --git a/include/kore/pgsql.h b/include/kore/pgsql.h index 94d6959..574bb84 100644 --- a/include/kore/pgsql.h +++ b/include/kore/pgsql.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2018 Joris Vink + * Copyright (c) 2014-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/include/kore/python_api.h b/include/kore/python_api.h index 23abd8e..9dfac78 100644 --- a/include/kore/python_api.h +++ b/include/kore/python_api.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2016 Stanislav Yudin - * Copyright (c) 2017-2018 Joris Vink + * Copyright (c) 2017-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index 7ec9afe..9c4872b 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Joris Vink + * Copyright (c) 2017-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,6 +21,7 @@ struct python_coro { u_int64_t id; int state; PyObject *obj; + PyObject *result; struct pysocket_op *sockop; struct pygather_op *gatherop; struct http_request *request; @@ -147,13 +148,28 @@ static PyTypeObject pytimer_type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, }; +/* XXX */ +struct pysocket; +struct pysocket_op; + +struct pysocket_event { + struct kore_event evt; + struct pysocket *s; +}; + struct pysocket { PyObject_HEAD int fd; int family; int protocol; + int scheduled; PyObject *socket; socklen_t addr_len; + + struct pysocket_event event; + struct pysocket_op *recvop; + struct pysocket_op *sendop; + union { struct sockaddr_in ipv4; struct sockaddr_un sun; @@ -198,23 +214,22 @@ static PyTypeObject pysocket_type = { #define PYSOCKET_TYPE_RECVFROM 5 #define PYSOCKET_TYPE_SENDTO 6 -struct pysocket_data { - struct kore_event evt; - int fd; - int eof; - int type; - void *self; - struct python_coro *coro; - int state; - size_t length; - struct kore_buf buffer; - struct sockaddr_in sendaddr; - struct pysocket *socket; -}; - struct pysocket_op { PyObject_HEAD - struct pysocket_data data; + int eof; + int type; + void *self; + struct python_coro *coro; + int state; + size_t length; + struct kore_buf buffer; + struct pysocket *socket; + struct kore_timer *timer; + + union { + struct sockaddr_in ipv4; + struct sockaddr_un sun; + } sendaddr; }; static void pysocket_op_dealloc(struct pysocket_op *); diff --git a/src/accesslog.c b/src/accesslog.c index 0bda2ac..807b8ec 100644 --- a/src/accesslog.c +++ b/src/accesslog.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include diff --git a/src/auth.c b/src/auth.c index 51e059b..df3db8d 100644 --- a/src/auth.c +++ b/src/auth.c @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/src/bsd.c b/src/bsd.c index 87446b4..9b0d5eb 100644 --- a/src/bsd.c +++ b/src/bsd.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/src/buf.c b/src/buf.c index 291016e..af58983 100644 --- a/src/buf.c +++ b/src/buf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include + #include #include #include diff --git a/src/cli.c b/src/cli.c index 8086dfe..0170386 100644 --- a/src/cli.c +++ b/src/cli.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2018 Joris Vink + * Copyright (c) 2014-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/src/config.c b/src/config.c index 9c5f923..613071f 100644 --- a/src/config.c +++ b/src/config.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include #include diff --git a/src/connection.c b/src/connection.c index 28d1b33..b30f569 100644 --- a/src/connection.c +++ b/src/connection.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -281,6 +281,7 @@ kore_connection_handle(struct connection *c) switch (r) { case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: + kore_connection_start_idletimer(c); return (KORE_RESULT_OK); default: kore_debug("SSL_accept(): %s", ssl_errno_s); @@ -309,6 +310,7 @@ kore_connection_handle(struct connection *c) listener = (struct listener *)c->owner; if (listener->connect != NULL) { kore_runtime_connect(listener->connect, c); + kore_connection_start_idletimer(c); return (KORE_RESULT_OK); } } diff --git a/src/domain.c b/src/domain.c index 9fe5cf1..642f67e 100644 --- a/src/domain.c +++ b/src/domain.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,6 +21,7 @@ */ #include +#include #if !defined(KORE_NO_TLS) #include @@ -156,8 +157,11 @@ kore_domain_init(void) #endif #if !defined(TLS1_3_VERSION) - kore_log(LOG_NOTICE, "%s has no TLS 1.3 - will only use TLS 1.2", - OPENSSL_VERSION_TEXT); + if (!kore_quiet) { + kore_log(LOG_NOTICE, + "%s has no TLS 1.3 - will only use TLS 1.2", + OPENSSL_VERSION_TEXT); + } #endif #endif diff --git a/src/filemap.c b/src/filemap.c index 80ed2ea..6ed2a6a 100644 --- a/src/filemap.c +++ b/src/filemap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Joris Vink + * Copyright (c) 2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/src/fileref.c b/src/fileref.c index 6ff226b..f9fb38d 100644 --- a/src/fileref.c +++ b/src/fileref.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Joris Vink + * Copyright (c) 2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/src/http.c b/src/http.c index 12ec4fd..912de12 100644 --- a/src/http.c +++ b/src/http.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include #include diff --git a/src/keymgr.c b/src/keymgr.c index f9305fd..915f58a 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Joris Vink + * Copyright (c) 2017-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -60,7 +60,6 @@ struct key { char *rand_file = NULL; static TAILQ_HEAD(, key) keys; -extern volatile sig_atomic_t sig_recv; static int initialized = 0; static void keymgr_reload(void); @@ -107,7 +106,7 @@ kore_keymgr_run(void) if (rand_file != NULL) { keymgr_load_randfile(); keymgr_save_randfile(); - } else { + } else if (!kore_quiet) { kore_log(LOG_WARNING, "no rand_file location specified"); } @@ -122,7 +121,8 @@ kore_keymgr_run(void) fatal("failed to pledge keymgr process"); #endif - kore_log(LOG_NOTICE, "key manager started"); + if (!kore_quiet) + kore_log(LOG_NOTICE, "key manager started"); while (quit != 1) { now = kore_time_ms(); @@ -162,7 +162,7 @@ kore_keymgr_cleanup(int final) { struct key *key, *next; - if (final) + if (final && !kore_quiet) kore_log(LOG_NOTICE, "cleaning up keys"); if (initialized == 0) @@ -182,7 +182,8 @@ keymgr_reload(void) { struct kore_domain *dom; - kore_log(LOG_INFO, "(re)loading certificates, keys and CRLs"); + if (!kore_quiet) + kore_log(LOG_INFO, "(re)loading certificates, keys and CRLs"); kore_keymgr_cleanup(0); TAILQ_INIT(&keys); diff --git a/src/kore.c b/src/kore.c index 7a3913b..06534dd 100644 --- a/src/kore.c +++ b/src/kore.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/src/linux.c b/src/linux.c index 04a546a..9d6d8c7 100644 --- a/src/linux.c +++ b/src/linux.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/src/mem.c b/src/mem.c index abd5a03..86c4a5c 100644 --- a/src/mem.c +++ b/src/mem.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include #include diff --git a/src/module.c b/src/module.c index 5e9f4fd..9b6f7e1 100644 --- a/src/module.c +++ b/src/module.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include diff --git a/src/msg.c b/src/msg.c index ec0de7a..e124e0f 100644 --- a/src/msg.c +++ b/src/msg.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Joris Vink + * Copyright (c) 2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include diff --git a/src/net.c b/src/net.c index 9770c8a..478b36a 100644 --- a/src/net.c +++ b/src/net.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #if defined(__linux__) #include diff --git a/src/pgsql.c b/src/pgsql.c index a1828ea..0f10a6f 100644 --- a/src/pgsql.c +++ b/src/pgsql.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2018 Joris Vink + * Copyright (c) 2014-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/src/pool.c b/src/pool.c index b21ea85..212a1d9 100644 --- a/src/pool.c +++ b/src/pool.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include diff --git a/src/python.c b/src/python.c index 37a324d..c63c2cb 100644 --- a/src/python.c +++ b/src/python.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2016 Stanislav Yudin - * Copyright (c) 2017-2018 Joris Vink + * Copyright (c) 2017-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,9 +17,12 @@ #include #include +#include #include #include +#include +#include #include #include @@ -37,6 +40,7 @@ static PyMODINIT_FUNC python_module_init(void); static PyObject *python_import(const char *); static PyObject *pyconnection_alloc(struct connection *); static PyObject *python_callable(PyObject *, const char *); +static void python_split_arguments(char *, char **, size_t); static int pyhttp_response_sent(struct netbuf *); static PyObject *pyhttp_file_alloc(struct http_file *); @@ -48,6 +52,7 @@ static int python_coro_run(struct python_coro *); static void python_coro_wakeup(struct python_coro *); static void pysocket_evt_handle(void *, int); +static void pysocket_op_timeout(void *, u_int64_t); static PyObject *pysocket_op_create(struct pysocket *, int, const void *, size_t); @@ -75,6 +80,7 @@ static void python_append_path(const char *); static void python_push_integer(PyObject *, const char *, long); static void python_push_type(const char *, PyObject *, PyTypeObject *); +static int python_validator_check(PyObject *); static int python_runtime_http_request(void *, struct http_request *); static int python_runtime_validator(void *, struct http_request *, const void *); @@ -281,6 +287,7 @@ kore_python_coro_delete(void *obj) else TAILQ_REMOVE(&coro_suspended, coro, list); + Py_XDECREF(coro->result); kore_pool_put(&coro_pool, coro); } @@ -413,6 +420,53 @@ python_module_free(struct kore_module *module) kore_free(module); } +static void +python_split_arguments(char *args, char **argv, size_t elm) +{ + size_t idx; + char *p, *line, *end; + + if (elm <= 1) + fatal("not enough elements (%zu)", elm); + + idx = 0; + line = args; + + for (p = line; *p != '\0'; p++) { + if (idx >= elm - 1) + break; + + if (*p == ' ') { + *p = '\0'; + if (*line != '\0') + argv[idx++] = line; + line = p + 1; + continue; + } + + if (*p != '"') + continue; + + line = p + 1; + if ((end = strchr(line, '"')) == NULL) + break; + + *end = '\0'; + argv[idx++] = line; + line = end + 1; + + while (isspace(*(unsigned char *)line)) + line++; + + p = line; + } + + if (idx < elm - 1 && *line != '\0') + argv[idx++] = line; + + argv[idx] = NULL; +} + static void python_module_reload(struct kore_module *module) { @@ -453,6 +507,7 @@ python_coro_create(PyObject *obj, struct http_request *req) coro = kore_pool_get(&coro_pool); coro_count++; + coro->result = NULL; coro->sockop = NULL; coro->gatherop = NULL; coro->exception = NULL; @@ -475,6 +530,7 @@ static int python_coro_run(struct python_coro *coro) { PyObject *item; + PyObject *type, *traceback; if (coro->state != CORO_STATE_RUNNABLE) fatal("non-runnable coro attempted to run"); @@ -486,7 +542,15 @@ python_coro_run(struct python_coro *coro) item = _PyGen_Send((PyGenObject *)coro->obj, NULL); if (item == NULL) { - kore_python_log_error("coroutine"); + if (PyErr_Occurred() && + PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Fetch(&type, &coro->result, &traceback); + Py_DECREF(type); + Py_XDECREF(traceback); + } else { + kore_python_log_error("coroutine"); + } + coro_running = NULL; return (KORE_RESULT_OK); } @@ -599,29 +663,49 @@ python_runtime_http_request(void *addr, struct http_request *req) static int python_runtime_validator(void *addr, struct http_request *req, const void *data) { - int ret; - PyObject *pyret, *pyreq, *args, *callable, *arg; + int ret; + struct python_coro *coro; + PyObject *pyret, *pyreq, *args, *callable, *arg; + + if (req->py_coro != NULL) { + coro = req->py_coro; + python_coro_wakeup(coro); + if (python_coro_run(coro) == KORE_RESULT_OK) { + ret = python_validator_check(coro->result); + kore_python_coro_delete(coro); + req->py_coro = NULL; + return (ret); + } + + return (KORE_RESULT_RETRY); + } callable = (PyObject *)addr; - if ((pyreq = pyhttp_request_alloc(req)) == NULL) - fatal("python_runtime_validator: pyreq alloc failed"); - if (req->flags & HTTP_VALIDATOR_IS_REQUEST) { if ((arg = pyhttp_request_alloc(data)) == NULL) - fatal("python_runtime_validator: pyreq failed"); + fatal("%s: pyreq failed", __func__); + + if ((args = PyTuple_New(1)) == NULL) + fatal("%s: PyTuple_New failed", __func__); + + if (PyTuple_SetItem(args, 0, arg) != 0) + fatal("%s: PyTuple_SetItem failed", __func__); } else { + if ((pyreq = pyhttp_request_alloc(req)) == NULL) + fatal("%s: pyreq alloc failed", __func__); + if ((arg = PyUnicode_FromString(data)) == NULL) fatal("python_runtime_validator: PyUnicode failed"); + + if ((args = PyTuple_New(2)) == NULL) + fatal("%s: PyTuple_New failed", __func__); + + if (PyTuple_SetItem(args, 0, pyreq) != 0 || + PyTuple_SetItem(args, 1, arg) != 0) + fatal("%s: PyTuple_SetItem failed", __func__); } - if ((args = PyTuple_New(2)) == NULL) - fatal("python_runtime_validator: PyTuple_New failed"); - - if (PyTuple_SetItem(args, 0, pyreq) != 0 || - PyTuple_SetItem(args, 1, arg) != 0) - fatal("python_runtime_vaildator: PyTuple_SetItem failed"); - PyErr_Clear(); pyret = PyObject_Call(callable, args, NULL); Py_DECREF(args); @@ -631,15 +715,55 @@ python_runtime_validator(void *addr, struct http_request *req, const void *data) fatal("failed to execute python call"); } - if (!PyLong_Check(pyret)) - fatal("python_runtime_validator: unexpected return type"); + if (PyCoro_CheckExact(pyret)) { + coro = python_coro_create(pyret, req); + req->py_coro = coro; + if (python_coro_run(coro) == KORE_RESULT_OK) { + ret = python_validator_check(coro->result); + kore_python_coro_delete(coro); + req->py_coro = NULL; + return (ret); + } + http_request_sleep(req); + return (KORE_RESULT_RETRY); + } - ret = (int)PyLong_AsLong(pyret); + ret = python_validator_check(pyret); Py_DECREF(pyret); return (ret); } +static int +python_validator_check(PyObject *obj) +{ + int ret; + + if (obj == NULL) + return (KORE_RESULT_ERROR); + + if (!PyLong_Check(obj)) { + kore_log(LOG_WARNING, + "invalid return value from authenticator (not an int)"); + ret = KORE_RESULT_ERROR; + } else { + ret = (int)PyLong_AsLong(obj); + } + + switch (ret) { + case KORE_RESULT_OK: + case KORE_RESULT_ERROR: + break; + default: + kore_log(LOG_WARNING, + "unsupported authenticator return value '%d'", ret); + ret = KORE_RESULT_ERROR; + break; + } + + return (ret); +} + static void python_runtime_wsmessage(void *addr, struct connection *c, u_int8_t op, const void *data, size_t len) @@ -1221,7 +1345,7 @@ python_kore_proc(PyObject *self, PyObject *args) { const char *cmd; struct pyproc *proc; - char *copy, *argv[30]; + char *copy, *argv[32]; int timeo, in_pipe[2], out_pipe[2]; timeo = -1; @@ -1290,9 +1414,9 @@ python_kore_proc(PyObject *self, PyObject *args) fatal("dup2: %s", errno_s); copy = kore_strdup(cmd); - kore_split_string(copy, " ", argv, 30); + python_split_arguments(copy, argv, 32); (void)execve(argv[0], argv, NULL); - printf("kore.proc failed to execute %s (%s)\n", + kore_log(LOG_ERR, "kore.proc failed to execute %s (%s)", argv[0], errno_s); exit(1); } @@ -1318,9 +1442,16 @@ python_kore_proc(PyObject *self, PyObject *args) static PyObject * python_import(const char *path) { + struct stat st; PyObject *module; char *dir, *file, *copy, *p; + if (stat(path, &st) == -1) + fatal("python_import: stat(%s): %s", path, errno_s); + + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) + fatal("python_import: '%s' is not a file or directory", path); + copy = kore_strdup(path); if ((file = basename(copy)) == NULL) @@ -1332,6 +1463,10 @@ python_import(const char *path) *p = '\0'; python_append_path(dir); + + if (S_ISDIR(st.st_mode)) + python_append_path(path); + module = PyImport_ImportModule(file); if (module == NULL) PyErr_Print(); @@ -1589,7 +1724,16 @@ pysocket_alloc(void) sock->fd = -1; sock->family = -1; sock->protocol = -1; + sock->scheduled = 0; + sock->socket = NULL; + sock->recvop = NULL; + sock->sendop = NULL; + + sock->event.s = sock; + sock->event.evt.flags = 0; + sock->event.evt.type = KORE_TYPE_PYSOCKET; + sock->event.evt.handle = pysocket_evt_handle; return (sock); } @@ -1597,6 +1741,13 @@ pysocket_alloc(void) static void pysocket_dealloc(struct pysocket *sock) { + if (sock->scheduled && sock->fd == -1) { + kore_platform_disable_read(sock->fd); +#if !defined(__linux__) + kore_platform_disable_write(sock->fd); +#endif + } + if (sock->socket != NULL) { Py_DECREF(sock->socket); } else if (sock->fd != -1) { @@ -1622,21 +1773,25 @@ pysocket_sendto(struct pysocket *sock, PyObject *args) { Py_buffer buf; struct pysocket_op *op; - const char *ip; PyObject *ret; int port; + const char *ip, *sockaddr; - if (sock->family != AF_INET) { - PyErr_SetString(PyExc_RuntimeError, - "sendto only supported on AF_INET sockets"); - return (NULL); - } - - if (!PyArg_ParseTuple(args, "siy*", &ip, &port, &buf)) - return (NULL); - - if (port <= 0 || port >= USHRT_MAX) { - PyErr_SetString(PyExc_RuntimeError, "invalid port"); + switch (sock->family) { + case AF_INET: + if (!PyArg_ParseTuple(args, "siy*", &ip, &port, &buf)) + return (NULL); + if (port <= 0 || port >= USHRT_MAX) { + PyErr_SetString(PyExc_RuntimeError, "invalid port"); + return (NULL); + } + break; + case AF_UNIX: + if (!PyArg_ParseTuple(args, "sy*", &sockaddr, &buf)) + return (NULL); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "unsupported family"); return (NULL); } @@ -1644,9 +1799,28 @@ pysocket_sendto(struct pysocket *sock, PyObject *args) op = (struct pysocket_op *)ret; - op->data.sendaddr.sin_family = AF_INET; - op->data.sendaddr.sin_port = htons(port); - op->data.sendaddr.sin_addr.s_addr = inet_addr(ip); + switch (sock->family) { + case AF_INET: + op->sendaddr.ipv4.sin_family = AF_INET; + op->sendaddr.ipv4.sin_port = htons(port); + op->sendaddr.ipv4.sin_addr.s_addr = inet_addr(ip); + break; + case AF_UNIX: + op->sendaddr.sun.sun_family = AF_UNIX; + if (kore_strlcpy(op->sendaddr.sun.sun_path, sockaddr, + sizeof(op->sendaddr.sun.sun_path)) >= + sizeof(op->sendaddr.sun.sun_path)) { + Py_DECREF(ret); + PyErr_SetString(PyExc_RuntimeError, + "unix socket path too long"); + return (NULL); + } + break; + default: + Py_DECREF(ret); + PyErr_SetString(PyExc_RuntimeError, "unsupported family"); + return (NULL); + } return (ret); } @@ -1654,12 +1828,28 @@ pysocket_sendto(struct pysocket *sock, PyObject *args) static PyObject * pysocket_recv(struct pysocket *sock, PyObject *args) { - Py_ssize_t len; + Py_ssize_t len; + struct pysocket_op *op; + PyObject *obj; + int timeo; - if (!PyArg_ParseTuple(args, "n", &len)) + timeo = -1; + + if (!PyArg_ParseTuple(args, "n|i", &len, &timeo)) return (NULL); - return (pysocket_op_create(sock, PYSOCKET_TYPE_RECV, NULL, len)); + obj = pysocket_op_create(sock, PYSOCKET_TYPE_RECV, NULL, len); + if (obj == NULL) + return (NULL); + + op = (struct pysocket_op *)obj; + + if (timeo != -1) { + op->timer = kore_timer_add(pysocket_op_timeout, + timeo, op, KORE_TIMER_ONESHOT); + } + + return (obj); } static PyObject * @@ -1667,12 +1857,6 @@ pysocket_recvfrom(struct pysocket *sock, PyObject *args) { Py_ssize_t len; - if (sock->family != AF_INET) { - PyErr_SetString(PyExc_RuntimeError, - "recvfrom only supported on AF_INET sockets"); - return (NULL); - } - if (!PyArg_ParseTuple(args, "n", &len)) return (NULL); @@ -1738,6 +1922,14 @@ pysocket_connect(struct pysocket *sock, PyObject *args) static PyObject * pysocket_close(struct pysocket *sock, PyObject *args) { + if (sock->scheduled) { + sock->scheduled = 0; + kore_platform_disable_read(sock->fd); +#if !defined(__linux__) + kore_platform_disable_write(sock->fd); +#endif + } + if (sock->socket != NULL) { Py_DECREF(sock->socket); sock->socket = NULL; @@ -1752,33 +1944,35 @@ pysocket_close(struct pysocket *sock, PyObject *args) static void pysocket_op_dealloc(struct pysocket_op *op) { -#if defined(__linux__) - kore_platform_disable_read(op->data.fd); - close(op->data.fd); -#else - switch (op->data.type) { + if (op->type == PYSOCKET_TYPE_RECV || + op->type == PYSOCKET_TYPE_RECVFROM || + op->type == PYSOCKET_TYPE_SEND) + kore_buf_cleanup(&op->buffer); + + switch (op->type) { case PYSOCKET_TYPE_RECV: case PYSOCKET_TYPE_ACCEPT: case PYSOCKET_TYPE_RECVFROM: - kore_platform_disable_read(op->data.fd); + if (op->socket->recvop != op) + fatal("recvop mismatch"); + op->socket->recvop = NULL; break; case PYSOCKET_TYPE_SEND: case PYSOCKET_TYPE_SENDTO: case PYSOCKET_TYPE_CONNECT: - kore_platform_disable_write(op->data.fd); + if (op->socket->sendop != op) + fatal("sendop mismatch"); + op->socket->sendop = NULL; break; - default: - fatal("unknown pysocket_op type %u", op->data.type); } -#endif - if (op->data.type == PYSOCKET_TYPE_RECV || - op->data.type == PYSOCKET_TYPE_RECVFROM || - op->data.type == PYSOCKET_TYPE_SEND) - kore_buf_cleanup(&op->data.buffer); + if (op->timer != NULL) { + kore_timer_remove(op->timer); + op->timer = NULL; + } - op->data.coro->sockop = NULL; - Py_DECREF(op->data.socket); + op->coro->sockop = NULL; + Py_DECREF(op->socket); PyObject_Del((PyObject *)op); } @@ -1791,62 +1985,71 @@ pysocket_op_create(struct pysocket *sock, int type, const void *ptr, size_t len) if (coro_running->sockop != NULL) fatal("pysocket_op_create: coro has active socketop"); + switch (type) { + case PYSOCKET_TYPE_RECV: + case PYSOCKET_TYPE_ACCEPT: + case PYSOCKET_TYPE_RECVFROM: + if (sock->recvop != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "only one recv operation can be done per socket"); + return (NULL); + } + break; + case PYSOCKET_TYPE_SEND: + case PYSOCKET_TYPE_SENDTO: + case PYSOCKET_TYPE_CONNECT: + if (sock->sendop != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "only one send operation can be done per socket"); + return (NULL); + } + break; + default: + fatal("unknown pysocket_op type %u", type); + } + op = PyObject_New(struct pysocket_op, &pysocket_op_type); if (op == NULL) return (NULL); -#if defined(__linux__) - /* - * Duplicate the socket so each pysocket_op gets its own unique - * descriptor for epoll. This is so we can easily call EPOLL_CTL_DEL - * on the fd when the pysocket_op is finished as our event code - * does not track queued events. - */ - if ((op->data.fd = dup(sock->fd)) == -1) - fatal("dup: %s", errno_s); -#else - op->data.fd = sock->fd; -#endif - - op->data.eof = 0; - op->data.self = op; - op->data.type = type; - op->data.socket = sock; - op->data.evt.flags = 0; - op->data.coro = coro_running; - op->data.evt.type = KORE_TYPE_PYSOCKET; - op->data.evt.handle = pysocket_evt_handle; + op->eof = 0; + op->self = op; + op->type = type; + op->timer = NULL; + op->socket = sock; + op->coro = coro_running; coro_running->sockop = op; - Py_INCREF(op->data.socket); + Py_INCREF(op->socket); switch (type) { case PYSOCKET_TYPE_RECV: case PYSOCKET_TYPE_RECVFROM: - op->data.evt.flags |= KORE_EVENT_READ; - kore_buf_init(&op->data.buffer, len); - kore_platform_schedule_read(op->data.fd, &op->data); + sock->recvop = op; + kore_buf_init(&op->buffer, len); break; case PYSOCKET_TYPE_SEND: case PYSOCKET_TYPE_SENDTO: - op->data.evt.flags |= KORE_EVENT_WRITE; - kore_buf_init(&op->data.buffer, len); - kore_buf_append(&op->data.buffer, ptr, len); - kore_buf_reset(&op->data.buffer); - kore_platform_schedule_write(op->data.fd, &op->data); + sock->sendop = op; + kore_buf_init(&op->buffer, len); + kore_buf_append(&op->buffer, ptr, len); + kore_buf_reset(&op->buffer); break; case PYSOCKET_TYPE_ACCEPT: - op->data.evt.flags |= KORE_EVENT_READ; - kore_platform_schedule_read(op->data.fd, &op->data); + sock->recvop = op; break; case PYSOCKET_TYPE_CONNECT: - op->data.evt.flags |= KORE_EVENT_WRITE; - kore_platform_schedule_write(op->data.fd, &op->data); + sock->sendop = op; break; default: fatal("unknown pysocket_op type %u", type); } + if (sock->scheduled == 0) { + sock->scheduled = 1; + kore_platform_event_all(sock->fd, &sock->event); + } + return ((PyObject *)op); } @@ -1862,25 +2065,25 @@ pysocket_op_iternext(struct pysocket_op *op) { PyObject *ret; - if (op->data.eof) { - if (op->data.coro->exception != NULL) { - PyErr_SetString(op->data.coro->exception, - op->data.coro->exception_msg); - op->data.coro->exception = NULL; + if (op->eof) { + if (op->coro->exception != NULL) { + PyErr_SetString(op->coro->exception, + op->coro->exception_msg); + op->coro->exception = NULL; return (NULL); } - if (op->data.type != PYSOCKET_TYPE_RECV) { + if (op->type != PYSOCKET_TYPE_RECV) { PyErr_SetString(PyExc_RuntimeError, "socket EOF"); return (NULL); } /* Drain the recv socket. */ - op->data.evt.flags |= KORE_EVENT_READ; + op->socket->event.evt.flags |= KORE_EVENT_READ; return (pysocket_async_recv(op)); } - switch (op->data.type) { + switch (op->type) { case PYSOCKET_TYPE_CONNECT: ret = pysocket_async_connect(op); break; @@ -1903,13 +2106,30 @@ pysocket_op_iternext(struct pysocket_op *op) return (ret); } +static void +pysocket_op_timeout(void *arg, u_int64_t now) +{ + struct pysocket_op *op = arg; + + op->eof = 1; + op->timer = NULL; + + op->coro->exception = PyExc_TimeoutError; + op->coro->exception_msg = "timeout before operation completed"; + + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); +} + static PyObject * pysocket_async_connect(struct pysocket_op *op) { - if (connect(op->data.fd, (struct sockaddr *)&op->data.socket->addr, - op->data.socket->addr_len) == -1) { + if (connect(op->socket->fd, (struct sockaddr *)&op->socket->addr, + op->socket->addr_len) == -1) { if (errno != EALREADY && errno != EINPROGRESS && - errno != EISCONN) { + errno != EISCONN && errno != EAGAIN) { PyErr_SetString(PyExc_RuntimeError, errno_s); return (NULL); } @@ -1929,15 +2149,20 @@ pysocket_async_accept(struct pysocket_op *op) int fd; struct pysocket *sock; - if ((sock = pysocket_alloc() ) == NULL) + if (!(op->socket->event.evt.flags & KORE_EVENT_READ)) { + Py_RETURN_NONE; + } + + if ((sock = pysocket_alloc()) == NULL) return (NULL); sock->addr_len = sizeof(sock->addr); - if ((fd = accept(op->data.fd, + if ((fd = accept(op->socket->fd, (struct sockaddr *)&sock->addr, &sock->addr_len)) == -1) { Py_DECREF((PyObject *)sock); if (errno == EAGAIN || errno == EWOULDBLOCK) { + op->socket->event.evt.flags &= ~KORE_EVENT_READ; Py_RETURN_NONE; } PyErr_SetString(PyExc_RuntimeError, errno_s); @@ -1952,8 +2177,8 @@ pysocket_async_accept(struct pysocket_op *op) sock->fd = fd; sock->socket = NULL; - sock->family = op->data.socket->family; - sock->protocol = op->data.socket->protocol; + sock->family = op->socket->family; + sock->protocol = op->socket->protocol; PyErr_SetObject(PyExc_StopIteration, (PyObject *)sock); Py_DECREF((PyObject *)sock); @@ -1967,30 +2192,40 @@ pysocket_async_recv(struct pysocket_op *op) ssize_t ret; u_int16_t port; socklen_t socklen; + struct sockaddr *sendaddr; const char *ptr, *ip; PyObject *bytes, *result, *tuple; - if (!(op->data.evt.flags & KORE_EVENT_READ)) { + if (!(op->socket->event.evt.flags & KORE_EVENT_READ)) { Py_RETURN_NONE; } for (;;) { - if (op->data.type == PYSOCKET_TYPE_RECV) { - ret = read(op->data.fd, op->data.buffer.data, - op->data.buffer.length); + if (op->type == PYSOCKET_TYPE_RECV) { + ret = read(op->socket->fd, op->buffer.data, + op->buffer.length); } else { - socklen = sizeof(op->data.sendaddr); - ret = recvfrom(op->data.fd, op->data.buffer.data, - op->data.buffer.length, 0, - (struct sockaddr *)&op->data.sendaddr, - &socklen); + sendaddr = (struct sockaddr *)&op->sendaddr; + switch (op->socket->family) { + case AF_INET: + socklen = sizeof(op->sendaddr.ipv4); + break; + case AF_UNIX: + socklen = sizeof(op->sendaddr.sun); + break; + default: + fatal("non AF_INET/AF_UNIX in %s", __func__); + } + + ret = recvfrom(op->socket->fd, op->buffer.data, + op->buffer.length, 0, sendaddr, &socklen); } if (ret == -1) { if (errno == EINTR) continue; if (errno == EAGAIN || errno == EWOULDBLOCK) { - op->data.evt.flags &= ~KORE_EVENT_READ; + op->socket->event.evt.flags &= ~KORE_EVENT_READ; Py_RETURN_NONE; } PyErr_SetString(PyExc_RuntimeError, errno_s); @@ -2000,26 +2235,46 @@ pysocket_async_recv(struct pysocket_op *op) break; } - if (op->data.type == PYSOCKET_TYPE_RECV && ret == 0) { + op->coro->exception = NULL; + op->coro->exception_msg = NULL; + + if (op->timer != NULL) { + kore_timer_remove(op->timer); + op->timer = NULL; + } + + if (op->type == PYSOCKET_TYPE_RECV && ret == 0) { PyErr_SetNone(PyExc_StopIteration); return (NULL); } - ptr = (const char *)op->data.buffer.data; + ptr = (const char *)op->buffer.data; if ((bytes = PyBytes_FromStringAndSize(ptr, ret)) == NULL) return (NULL); - if (op->data.type == PYSOCKET_TYPE_RECV) { + if (op->type == PYSOCKET_TYPE_RECV) { PyErr_SetObject(PyExc_StopIteration, bytes); Py_DECREF(bytes); return (NULL); } - port = ntohs(op->data.sendaddr.sin_port); - ip = inet_ntoa(op->data.sendaddr.sin_addr); + switch(op->socket->family) { + case AF_INET: + port = ntohs(op->sendaddr.ipv4.sin_port); + ip = inet_ntoa(op->sendaddr.ipv4.sin_addr); - if ((tuple = Py_BuildValue("(sHN)", ip, port, bytes)) == NULL) + if ((tuple = Py_BuildValue("(sHN)", ip, port, bytes)) == NULL) + return (NULL); + break; + case AF_UNIX: + if ((tuple = Py_BuildValue("(sN)", + op->sendaddr.sun.sun_path, bytes)) == NULL) + return (NULL); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unsupported family"); return (NULL); + } result = PyObject_CallFunctionObjArgs(PyExc_StopIteration, tuple, NULL); if (result == NULL) { @@ -2036,30 +2291,45 @@ pysocket_async_recv(struct pysocket_op *op) static PyObject * pysocket_async_send(struct pysocket_op *op) { - ssize_t ret; + ssize_t ret; + socklen_t socklen; + const struct sockaddr *sendaddr; - if (!(op->data.evt.flags & KORE_EVENT_WRITE)) { + if (!(op->socket->event.evt.flags & KORE_EVENT_WRITE)) { Py_RETURN_NONE; } for (;;) { - if (op->data.type == PYSOCKET_TYPE_SEND) { - ret = write(op->data.fd, - op->data.buffer.data + op->data.buffer.offset, - op->data.buffer.length - op->data.buffer.offset); + if (op->type == PYSOCKET_TYPE_SEND) { + ret = write(op->socket->fd, + op->buffer.data + op->buffer.offset, + op->buffer.length - op->buffer.offset); } else { - ret = sendto(op->data.fd, - op->data.buffer.data + op->data.buffer.offset, - op->data.buffer.length - op->data.buffer.offset, - 0, (const struct sockaddr *)&op->data.sendaddr, - sizeof(op->data.sendaddr)); + sendaddr = (const struct sockaddr *)&op->sendaddr; + + switch (op->socket->family) { + case AF_INET: + socklen = sizeof(op->sendaddr.ipv4); + break; + case AF_UNIX: + socklen = sizeof(op->sendaddr.sun); + break; + default: + fatal("non AF_INET/AF_UNIX in %s", __func__); + } + + ret = sendto(op->socket->fd, + op->buffer.data + op->buffer.offset, + op->buffer.length - op->buffer.offset, + 0, sendaddr, socklen); } if (ret == -1) { if (errno == EINTR) continue; if (errno == EAGAIN || errno == EWOULDBLOCK) { - op->data.evt.flags &= ~KORE_EVENT_WRITE; + op->socket->event.evt.flags &= + ~KORE_EVENT_WRITE; Py_RETURN_NONE; } PyErr_SetString(PyExc_RuntimeError, errno_s); @@ -2068,9 +2338,9 @@ pysocket_async_send(struct pysocket_op *op) break; } - op->data.buffer.offset += (size_t)ret; + op->buffer.offset += (size_t)ret; - if (op->data.buffer.offset == op->data.buffer.length) { + if (op->buffer.offset == op->buffer.length) { PyErr_SetNone(PyExc_StopIteration); return (NULL); } @@ -2081,25 +2351,26 @@ pysocket_async_send(struct pysocket_op *op) static void pysocket_evt_handle(void *arg, int eof) { - struct pysocket_data *data = arg; - struct python_coro *coro = data->coro; + struct pysocket_event *event = arg; + struct pysocket *socket = event->s; - if (coro->sockop == NULL) - fatal("pysocket_evt_handle: sockop == NULL"); + if ((eof || (event->evt.flags & KORE_EVENT_READ)) && + socket->recvop != NULL) { + if (socket->recvop->coro->request != NULL) + http_request_wakeup(socket->recvop->coro->request); + else + python_coro_wakeup(socket->recvop->coro); + socket->recvop->eof = eof; + } - /* - * If we are a coroutine tied to an HTTP request wake-up the - * HTTP request instead. That in turn will wakeup the coro and - * continue it. - * - * Otherwise just wakeup the coroutine so it will run next tick. - */ - if (coro->request != NULL) - http_request_wakeup(coro->request); - else - python_coro_wakeup(coro); - - coro->sockop->data.eof = eof; + if ((eof || (event->evt.flags & KORE_EVENT_WRITE)) && + socket->sendop != NULL) { + if (socket->sendop->coro->request != NULL) + http_request_wakeup(socket->sendop->coro->request); + else + python_coro_wakeup(socket->sendop->coro); + socket->sendop->eof = eof; + } } static void @@ -2258,7 +2529,7 @@ pylock_dealloc(struct pylock *lock) Py_DECREF((PyObject *)op); } - PyObject_Del((PyObject *)op); + PyObject_Del((PyObject *)lock); } static PyObject * @@ -2385,8 +2656,6 @@ pylock_op_iternext(struct pylock_op *op) TAILQ_REMOVE(&op->lock->ops, op, list); PyErr_SetNone(PyExc_StopIteration); - Py_DECREF((PyObject *)op); - return (NULL); } @@ -2398,7 +2667,7 @@ pyproc_timeout(void *arg, u_int64_t now) proc->timer = NULL; if (proc->coro->sockop != NULL) - proc->coro->sockop->data.eof = 1; + proc->coro->sockop->eof = 1; proc->coro->exception = PyExc_TimeoutError; proc->coro->exception_msg = "timeout before process exited"; @@ -2485,17 +2754,33 @@ pyproc_reap(struct pyproc *proc, PyObject *args) static PyObject * pyproc_recv(struct pyproc *proc, PyObject *args) { - Py_ssize_t len; + Py_ssize_t len; + struct pysocket_op *op; + PyObject *obj; + int timeo; + + timeo = -1; if (proc->out == NULL) { PyErr_SetString(PyExc_RuntimeError, "stdout closed"); return (NULL); } - if (!PyArg_ParseTuple(args, "n", &len)) + if (!PyArg_ParseTuple(args, "n|i", &len, &timeo)) return (NULL); - return (pysocket_op_create(proc->out, PYSOCKET_TYPE_RECV, NULL, len)); + obj = pysocket_op_create(proc->out, PYSOCKET_TYPE_RECV, NULL, len); + if (obj == NULL) + return (NULL); + + op = (struct pysocket_op *)obj; + + if (timeo != -1) { + op->timer = kore_timer_add(pysocket_op_timeout, + timeo, op, KORE_TIMER_ONESHOT); + } + + return (obj); } static PyObject * diff --git a/src/runtime.c b/src/runtime.c index 80591ce..dcb6c41 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Joris Vink + * Copyright (c) 2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include "kore.h" diff --git a/src/timer.c b/src/timer.c index 319fabd..8ba5f65 100644 --- a/src/timer.c +++ b/src/timer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2018 Joris Vink + * Copyright (c) 2016-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include #include "kore.h" diff --git a/src/utils.c b/src/utils.c index 647951b..489edae 100644 --- a/src/utils.c +++ b/src/utils.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include diff --git a/src/validator.c b/src/validator.c index d419254..673a670 100644 --- a/src/validator.c +++ b/src/validator.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include + #include "kore.h" TAILQ_HEAD(, kore_validator) validators; diff --git a/src/websocket.c b/src/websocket.c index ce51174..f6f5a33 100644 --- a/src/websocket.c +++ b/src/websocket.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2018 Joris Vink + * Copyright (c) 2014-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/src/worker.c b/src/worker.c index 4cd3624..956bb45 100644 --- a/src/worker.c +++ b/src/worker.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include #include #include @@ -88,7 +89,6 @@ static int worker_no_lock; static int shm_accept_key; static struct wlock *accept_lock; -extern volatile sig_atomic_t sig_recv; struct kore_worker *worker = NULL; u_int8_t worker_set_affinity = 1; u_int32_t worker_accept_threshold = 16; @@ -280,7 +280,7 @@ kore_worker_privdrop(const char *runas, const char *root) rl.rlim_cur = worker_rlimit_nofiles; rl.rlim_max = worker_rlimit_nofiles; if (setrlimit(RLIMIT_NOFILE, &rl) == -1) { - kore_log(LOG_ERR, "setrlimit(RLIMIT_NOFILE, %d): %s", + kore_log(LOG_ERR, "setrlimit(RLIMIT_NOFILE, %u): %s", worker_rlimit_nofiles, errno_s); }