kore/include/kore/kore.h

868 lines
24 KiB
C
Raw Normal View History

2013-04-17 22:34:27 +02:00
/*
2019-02-22 16:57:28 +01:00
* Copyright (c) 2013-2019 Joris Vink <joris@coders.se>
2013-04-17 22:34:27 +02:00
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef __H_KORE_H
#define __H_KORE_H
#if defined(__APPLE__)
#define daemon portability_is_king
#endif
2018-11-29 10:31:14 +01:00
#include <sys/param.h>
2019-03-14 19:20:46 +01:00
#include <sys/types.h>
#include <sys/time.h>
#include <sys/queue.h>
#include <sys/un.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#if !defined(KORE_NO_TLS)
#include <openssl/err.h>
#include <openssl/dh.h>
#include <openssl/ssl.h>
#endif
#include <errno.h>
#include <regex.h>
#include <stdarg.h>
#include <stdlib.h>
#include <stdio.h>
2019-01-11 11:32:33 +01:00
#include <signal.h>
#include <string.h>
#include <syslog.h>
#include <unistd.h>
2016-01-22 12:08:13 +01:00
#include <stdarg.h>
2015-04-01 13:25:10 +02:00
#if defined(__cplusplus)
extern "C" {
#endif
#if defined(__APPLE__)
#undef daemon
extern int daemon(int, int);
#define st_mtim st_mtimespec
#endif
#if !defined(KORE_NO_SENDFILE) && defined(KORE_NO_TLS)
#if defined(__MACH__) || defined(__FreeBSD_version) || defined(__linux__)
#define KORE_USE_PLATFORM_SENDFILE 1
#endif
#endif
#if defined(__OpenBSD__)
#define KORE_USE_PLATFORM_PLEDGE 1
#endif
2013-04-17 22:34:27 +02:00
#define KORE_RESULT_ERROR 0
#define KORE_RESULT_OK 1
#define KORE_RESULT_RETRY 2
2013-04-17 22:34:27 +02:00
#define KORE_TLS_VERSION_1_3 0
#define KORE_TLS_VERSION_1_2 1
#define KORE_TLS_VERSION_BOTH 2
#define KORE_WAIT_INFINITE (u_int64_t)-1
#define KORE_RESEED_TIME (1800 * 1000)
2013-04-17 22:34:27 +02:00
#define errno_s strerror(errno)
#define ssl_errno_s ERR_error_string(ERR_get_error(), NULL)
#define KORE_DOMAINNAME_LEN 255
#define KORE_PIDFILE_DEFAULT "kore.pid"
#define KORE_DEFAULT_CIPHER_LIST "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!kRSA:!kDSA"
#if defined(KORE_DEBUG)
#define kore_debug(...) \
if (kore_debug) \
2016-02-13 14:20:31 +01:00
kore_debug_internal(__FILE__, __LINE__, __VA_ARGS__)
#else
#define kore_debug(...)
#endif
2013-04-17 22:34:27 +02:00
#define NETBUF_RECV 0
#define NETBUF_SEND 1
#define NETBUF_SEND_PAYLOAD_MAX 8192
#define SENDFILE_PAYLOAD_MAX (1024 * 1024 * 10)
#define NETBUF_LAST_CHAIN 0
#define NETBUF_BEFORE_CHAIN 1
#define NETBUF_CALL_CB_ALWAYS 0x01
#define NETBUF_FORCE_REMOVE 0x02
#define NETBUF_MUST_RESEND 0x04
#define NETBUF_IS_STREAM 0x10
#define NETBUF_IS_FILEREF 0x20
#define X509_GET_CN(c, o, l) \
X509_NAME_get_text_by_NID(X509_get_subject_name(c), \
NID_commonName, o, l)
#define X509_CN_LENGTH (ub_common_name + 1)
/* XXX hackish. */
#if !defined(KORE_NO_HTTP)
struct http_request;
#endif
#define KORE_FILEREF_SOFT_REMOVED 0x1000
struct kore_fileref {
int cnt;
int flags;
off_t size;
char *path;
u_int64_t mtime;
time_t mtime_sec;
u_int64_t expiration;
#if !defined(KORE_USE_PLATFORM_SENDFILE)
void *base;
#else
int fd;
#endif
TAILQ_ENTRY(kore_fileref) list;
};
2013-04-17 22:34:27 +02:00
struct netbuf {
u_int8_t *buf;
size_t s_off;
size_t b_len;
size_t m_len;
u_int8_t type;
u_int8_t flags;
struct kore_fileref *file_ref;
#if defined(KORE_USE_PLATFORM_SENDFILE)
off_t fd_off;
off_t fd_len;
#endif
void *owner;
void *extra;
2013-04-17 22:34:27 +02:00
int (*cb)(struct netbuf *);
TAILQ_ENTRY(netbuf) list;
};
TAILQ_HEAD(netbuf_head, netbuf);
#define KORE_TYPE_LISTENER 1
#define KORE_TYPE_CONNECTION 2
#define KORE_TYPE_PGSQL_CONN 3
#define KORE_TYPE_TASK 4
#define KORE_TYPE_PYSOCKET 5
#define KORE_TYPE_CURL_HANDLE 6
#define CONN_STATE_UNKNOWN 0
2017-02-11 21:33:09 +01:00
#define CONN_STATE_TLS_SHAKE 1
#define CONN_STATE_ESTABLISHED 2
#define CONN_STATE_DISCONNECTING 3
#define CONN_PROTO_UNKNOWN 0
#define CONN_PROTO_HTTP 1
#define CONN_PROTO_WEBSOCKET 2
#define CONN_PROTO_MSG 3
#define KORE_EVENT_READ 0x01
#define KORE_EVENT_WRITE 0x02
#define KORE_EVENT_ERROR 0x04
#define CONN_IDLE_TIMER_ACT 0x01
#define CONN_CLOSE_EMPTY 0x02
#define CONN_WS_CLOSE_SENT 0x04
#define CONN_IS_BUSY 0x08
Rework HTTP and worker processes. The HTTP layer used to make a copy of each incoming header and its value for a request. Stop doing that and make HTTP headers zero-copy all across the board. This change comes with some api function changes, notably the http_request_header() function which now takes a const char ** rather than a char ** out pointer. This commit also constifies several members of http_request, beware. Additional rework how the worker processes deal with the accept lock. Before: if a worker held the accept lock and it accepted a new connection it would release the lock for others and back off for 500ms before attempting to grab the lock again. This approach worked but under high load this starts becoming obvious. Now: - workers not holding the accept lock and not having any connections will wait less long before returning from kore_platform_event_wait(). - workers not holding the accept lock will no longer blindly wait an arbitrary amount in kore_platform_event_wait() but will look at how long until the next lock grab is and base their timeout on that. - if a worker its next_lock timeout is up and failed to grab the lock it will try again in half the time again. - the worker process holding the lock will when releasing the lock double check if it still has space for newer connections, if it does it will keep the lock until it is full. This prevents the lock from bouncing between several non busy worker processes all the time. Additional fixes: - Reduce the number of times we check the timeout list, only do it twice per second rather then every event tick. - Fix solo worker count for TLS (we actually hold two processes, not one). - Make sure we don't accidentally miscalculate the idle time causing new connections under heavy load to instantly drop. - Swap from gettimeofday() to clock_gettime() now that MacOS caught up.
2018-02-14 13:48:49 +01:00
#define KORE_IDLE_TIMER_MAX 5000
#define WEBSOCKET_OP_CONT 0x00
#define WEBSOCKET_OP_TEXT 0x01
#define WEBSOCKET_OP_BINARY 0x02
#define WEBSOCKET_OP_CLOSE 0x08
#define WEBSOCKET_OP_PING 0x09
#define WEBSOCKET_OP_PONG 0x0a
#define WEBSOCKET_BROADCAST_LOCAL 1
#define WEBSOCKET_BROADCAST_GLOBAL 2
#define KORE_TIMER_ONESHOT 0x01
2018-10-23 20:44:43 +02:00
#define KORE_TIMER_FLAGS (KORE_TIMER_ONESHOT)
#define KORE_CONNECTION_PRUNE_DISCONNECT 0
#define KORE_CONNECTION_PRUNE_ALL 1
struct kore_event {
int type;
int flags;
void (*handle)(void *, int);
} __attribute__((packed));
2013-04-17 22:34:27 +02:00
struct connection {
struct kore_event evt;
2013-04-17 22:34:27 +02:00
int fd;
u_int8_t state;
u_int8_t proto;
2013-04-17 22:34:27 +02:00
void *owner;
#if !defined(KORE_NO_TLS)
X509 *cert;
SSL *ssl;
int tls_reneg;
#endif
u_int8_t flags;
void *hdlr_extra;
2013-04-17 22:34:27 +02:00
int (*handle)(struct connection *);
void (*disconnect)(struct connection *);
int (*read)(struct connection *, size_t *);
int (*write)(struct connection *, size_t, size_t *);
int family;
union {
struct sockaddr_in ipv4;
struct sockaddr_in6 ipv6;
2018-10-07 21:03:12 +02:00
struct sockaddr_un sun;
} addr;
struct {
u_int64_t length;
u_int64_t start;
} idle_timer;
struct netbuf_head send_queue;
struct netbuf *snb;
struct netbuf *rnb;
#if !defined(KORE_NO_HTTP)
u_int64_t http_start;
u_int64_t http_timeout;
struct kore_runtime_call *ws_connect;
struct kore_runtime_call *ws_message;
struct kore_runtime_call *ws_disconnect;
TAILQ_HEAD(, http_request) http_requests;
#endif
TAILQ_ENTRY(connection) list;
2013-04-17 22:34:27 +02:00
};
TAILQ_HEAD(connection_list, connection);
extern struct connection_list connections;
extern struct connection_list disconnected;
#define KORE_RUNTIME_NATIVE 0
#define KORE_RUNTIME_PYTHON 1
struct kore_runtime {
int type;
#if !defined(KORE_NO_HTTP)
int (*http_request)(void *, struct http_request *);
Rework HTTP and worker processes. The HTTP layer used to make a copy of each incoming header and its value for a request. Stop doing that and make HTTP headers zero-copy all across the board. This change comes with some api function changes, notably the http_request_header() function which now takes a const char ** rather than a char ** out pointer. This commit also constifies several members of http_request, beware. Additional rework how the worker processes deal with the accept lock. Before: if a worker held the accept lock and it accepted a new connection it would release the lock for others and back off for 500ms before attempting to grab the lock again. This approach worked but under high load this starts becoming obvious. Now: - workers not holding the accept lock and not having any connections will wait less long before returning from kore_platform_event_wait(). - workers not holding the accept lock will no longer blindly wait an arbitrary amount in kore_platform_event_wait() but will look at how long until the next lock grab is and base their timeout on that. - if a worker its next_lock timeout is up and failed to grab the lock it will try again in half the time again. - the worker process holding the lock will when releasing the lock double check if it still has space for newer connections, if it does it will keep the lock until it is full. This prevents the lock from bouncing between several non busy worker processes all the time. Additional fixes: - Reduce the number of times we check the timeout list, only do it twice per second rather then every event tick. - Fix solo worker count for TLS (we actually hold two processes, not one). - Make sure we don't accidentally miscalculate the idle time causing new connections under heavy load to instantly drop. - Swap from gettimeofday() to clock_gettime() now that MacOS caught up.
2018-02-14 13:48:49 +01:00
int (*validator)(void *, struct http_request *, const void *);
void (*wsconnect)(void *, struct connection *);
void (*wsdisconnect)(void *, struct connection *);
void (*wsmessage)(void *, struct connection *,
u_int8_t, const void *, size_t);
#endif
void (*execute)(void *);
int (*onload)(void *, int);
void (*connect)(void *, struct connection *);
void (*configure)(void *, int, char **);
};
struct kore_runtime_call {
void *addr;
struct kore_runtime *runtime;
};
extern struct kore_runtime kore_native_runtime;
struct listener {
struct kore_event evt;
int fd;
int family;
struct kore_runtime_call *connect;
LIST_ENTRY(listener) list;
};
LIST_HEAD(listener_head, listener);
#if !defined(KORE_NO_HTTP)
#define KORE_PARAMS_QUERY_STRING 0x0001
struct kore_handler_params {
char *name;
int flags;
u_int8_t method;
struct kore_validator *validator;
TAILQ_ENTRY(kore_handler_params) list;
};
#define KORE_AUTH_TYPE_COOKIE 1
#define KORE_AUTH_TYPE_HEADER 2
#define KORE_AUTH_TYPE_REQUEST 3
struct kore_auth {
u_int8_t type;
char *name;
char *value;
char *redirect;
struct kore_validator *validator;
TAILQ_ENTRY(kore_auth) list;
};
2013-05-01 16:03:48 +02:00
#define HANDLER_TYPE_STATIC 1
#define HANDLER_TYPE_DYNAMIC 2
#endif /* !KORE_NO_HTTP */
#define KORE_MODULE_LOAD 1
#define KORE_MODULE_UNLOAD 2
#define KORE_MODULE_NATIVE 0
#define KORE_MODULE_PYTHON 1
struct kore_module;
struct kore_module_functions {
void (*free)(struct kore_module *);
void (*reload)(struct kore_module *);
int (*callback)(struct kore_module *, int);
Rework HTTP and worker processes. The HTTP layer used to make a copy of each incoming header and its value for a request. Stop doing that and make HTTP headers zero-copy all across the board. This change comes with some api function changes, notably the http_request_header() function which now takes a const char ** rather than a char ** out pointer. This commit also constifies several members of http_request, beware. Additional rework how the worker processes deal with the accept lock. Before: if a worker held the accept lock and it accepted a new connection it would release the lock for others and back off for 500ms before attempting to grab the lock again. This approach worked but under high load this starts becoming obvious. Now: - workers not holding the accept lock and not having any connections will wait less long before returning from kore_platform_event_wait(). - workers not holding the accept lock will no longer blindly wait an arbitrary amount in kore_platform_event_wait() but will look at how long until the next lock grab is and base their timeout on that. - if a worker its next_lock timeout is up and failed to grab the lock it will try again in half the time again. - the worker process holding the lock will when releasing the lock double check if it still has space for newer connections, if it does it will keep the lock until it is full. This prevents the lock from bouncing between several non busy worker processes all the time. Additional fixes: - Reduce the number of times we check the timeout list, only do it twice per second rather then every event tick. - Fix solo worker count for TLS (we actually hold two processes, not one). - Make sure we don't accidentally miscalculate the idle time causing new connections under heavy load to instantly drop. - Swap from gettimeofday() to clock_gettime() now that MacOS caught up.
2018-02-14 13:48:49 +01:00
void (*load)(struct kore_module *);
void *(*getsym)(struct kore_module *, const char *);
};
struct kore_module {
void *handle;
char *path;
char *onload;
int type;
struct kore_runtime_call *ocb;
struct kore_module_functions *fun;
struct kore_runtime *runtime;
TAILQ_ENTRY(kore_module) list;
};
#if !defined(KORE_NO_HTTP)
struct kore_module_handle {
char *path;
char *func;
int type;
int errors;
regex_t rctx;
struct kore_domain *dom;
struct kore_runtime_call *rcall;
struct kore_auth *auth;
int methods;
TAILQ_HEAD(, kore_handler_params) params;
2013-05-01 16:03:48 +02:00
TAILQ_ENTRY(kore_module_handle) list;
};
#endif
2013-05-01 16:03:48 +02:00
/*
* The workers get a 128KB log buffer per worker, and parent will fetch their
* logs when it reached at least 75% of that or if its been > 1 second since
* it was last synced.
*/
2018-12-22 10:52:57 +01:00
#define KORE_ACCESSLOG_BUFLEN 131072U
#define KORE_ACCESSLOG_SYNC 98304U
struct kore_alog_header {
u_int16_t domain;
u_int16_t loglen;
} __attribute__((packed));
struct kore_worker {
u_int8_t id;
u_int8_t cpu;
pid_t pid;
int pipe[2];
struct connection *msg[2];
u_int8_t has_lock;
int restarted;
Rework HTTP and worker processes. The HTTP layer used to make a copy of each incoming header and its value for a request. Stop doing that and make HTTP headers zero-copy all across the board. This change comes with some api function changes, notably the http_request_header() function which now takes a const char ** rather than a char ** out pointer. This commit also constifies several members of http_request, beware. Additional rework how the worker processes deal with the accept lock. Before: if a worker held the accept lock and it accepted a new connection it would release the lock for others and back off for 500ms before attempting to grab the lock again. This approach worked but under high load this starts becoming obvious. Now: - workers not holding the accept lock and not having any connections will wait less long before returning from kore_platform_event_wait(). - workers not holding the accept lock will no longer blindly wait an arbitrary amount in kore_platform_event_wait() but will look at how long until the next lock grab is and base their timeout on that. - if a worker its next_lock timeout is up and failed to grab the lock it will try again in half the time again. - the worker process holding the lock will when releasing the lock double check if it still has space for newer connections, if it does it will keep the lock until it is full. This prevents the lock from bouncing between several non busy worker processes all the time. Additional fixes: - Reduce the number of times we check the timeout list, only do it twice per second rather then every event tick. - Fix solo worker count for TLS (we actually hold two processes, not one). - Make sure we don't accidentally miscalculate the idle time causing new connections under heavy load to instantly drop. - Swap from gettimeofday() to clock_gettime() now that MacOS caught up.
2018-02-14 13:48:49 +01:00
u_int64_t time_locked;
struct kore_module_handle *active_hdlr;
/* Used by the workers to store accesslogs. */
struct {
int lock;
size_t offset;
char buf[KORE_ACCESSLOG_BUFLEN];
} lb;
};
struct kore_domain {
u_int16_t id;
2013-06-24 09:36:40 +02:00
char *domain;
int accesslog;
struct kore_buf *logbuf;
int logerr;
u_int64_t logwarn;
#if !defined(KORE_NO_TLS)
2013-12-14 16:31:07 +01:00
char *cafile;
char *crlfile;
char *certfile;
char *certkey;
SSL_CTX *ssl_ctx;
int x509_verify_depth;
#endif
#if !defined(KORE_NO_HTTP)
2013-06-24 09:36:40 +02:00
TAILQ_HEAD(, kore_module_handle) handlers;
#endif
TAILQ_ENTRY(kore_domain) list;
2013-06-24 09:36:40 +02:00
};
TAILQ_HEAD(kore_domain_h, kore_domain);
#if !defined(KORE_NO_HTTP)
#define KORE_VALIDATOR_TYPE_REGEX 1
#define KORE_VALIDATOR_TYPE_FUNCTION 2
struct kore_validator {
u_int8_t type;
char *name;
char *arg;
regex_t rctx;
struct kore_runtime_call *rcall;
TAILQ_ENTRY(kore_validator) list;
};
#endif /* !KORE_NO_HTTP */
#define KORE_BUF_OWNER_API 0x0001
struct kore_buf {
u_int8_t *data;
int flags;
size_t length;
size_t offset;
};
struct kore_pool_region {
void *start;
size_t length;
LIST_ENTRY(kore_pool_region) list;
};
struct kore_pool_entry {
u_int8_t state;
struct kore_pool_region *region;
LIST_ENTRY(kore_pool_entry) list;
};
struct kore_pool {
2016-07-12 14:01:02 +02:00
size_t elen;
size_t slen;
size_t elms;
size_t inuse;
size_t growth;
volatile int lock;
char *name;
LIST_HEAD(, kore_pool_region) regions;
LIST_HEAD(, kore_pool_entry) freelist;
};
struct kore_timer {
u_int64_t nextrun;
u_int64_t interval;
int flags;
void *arg;
void (*cb)(void *, u_int64_t);
TAILQ_ENTRY(kore_timer) list;
};
#define KORE_WORKER_KEYMGR 0
#define KORE_WORKER_POLICY_RESTART 1
#define KORE_WORKER_POLICY_TERMINATE 2
/* Reserved message ids, registered on workers. */
#define KORE_MSG_WEBSOCKET 1
#define KORE_MSG_KEYMGR_REQ 2
#define KORE_MSG_KEYMGR_RESP 3
#define KORE_MSG_SHUTDOWN 4
2018-12-22 10:22:59 +01:00
#define KORE_MSG_ENTROPY_REQ 5
#define KORE_MSG_ENTROPY_RESP 6
#define KORE_MSG_CERTIFICATE 7
#define KORE_MSG_CERTIFICATE_REQ 8
#define KORE_MSG_CRL 9
#define KORE_MSG_ACCEPT_AVAILABLE 10
/* Predefined message targets. */
#define KORE_MSG_PARENT 1000
#define KORE_MSG_WORKER_ALL 1001
struct kore_msg {
u_int8_t id;
u_int16_t src;
u_int16_t dst;
size_t length;
};
#if !defined(KORE_NO_TLS)
struct kore_keyreq {
int padding;
char domain[KORE_DOMAINNAME_LEN];
u_int16_t domain_len;
u_int16_t data_len;
u_int8_t data[];
};
struct kore_x509_msg {
char domain[KORE_DOMAINNAME_LEN];
u_int16_t domain_len;
size_t data_len;
u_int8_t data[];
};
#endif
#if !defined(KORE_SINGLE_BINARY)
extern char *config_file;
#endif
2013-06-26 11:18:32 +02:00
extern pid_t kore_pid;
2014-07-31 13:43:37 +02:00
extern int foreground;
extern int kore_quiet;
extern int kore_debug;
extern int skip_chroot;
extern int skip_runas;
extern char *kore_pidfile;
extern char *kore_root_path;
extern char *kore_runas_user;
extern char *kore_tls_cipher_list;
extern volatile sig_atomic_t sig_recv;
#if !defined(KORE_NO_TLS)
extern int tls_version;
extern DH *tls_dhparam;
extern char *rand_file;
extern char *keymgr_runas_user;
extern char *keymgr_root_path;
#endif
extern u_int8_t nlisteners;
extern u_int16_t cpu_count;
extern u_int8_t worker_count;
extern const char *kore_version;
extern int worker_policy;
extern u_int8_t worker_set_affinity;
extern u_int32_t worker_rlimit_nofiles;
extern u_int32_t worker_max_connections;
extern u_int32_t worker_active_connections;
extern u_int32_t worker_accept_threshold;
extern u_int64_t kore_websocket_maxframe;
extern u_int64_t kore_websocket_timeout;
extern u_int32_t kore_socket_backlog;
extern struct listener_head listeners;
2013-06-24 09:36:40 +02:00
extern struct kore_worker *worker;
extern struct kore_domain_h domains;
extern struct kore_domain *primary_dom;
extern struct kore_pool nb_pool;
2013-06-26 11:18:32 +02:00
void kore_signal(int);
void kore_shutdown(void);
void kore_signal_setup(void);
void kore_proctitle(const char *);
void kore_worker_reap(void);
void kore_worker_init(void);
void kore_worker_make_busy(void);
void kore_worker_shutdown(void);
void kore_worker_dispatch_signal(int);
void kore_worker_spawn(u_int16_t, u_int16_t);
void kore_worker_entry(struct kore_worker *);
void kore_worker_privdrop(const char *, const char *);
struct kore_worker *kore_worker_data(u_int8_t);
2013-06-26 11:18:32 +02:00
void kore_platform_init(void);
void kore_platform_sandbox(void);
2013-06-26 11:18:32 +02:00
void kore_platform_event_init(void);
2016-01-04 22:40:14 +01:00
void kore_platform_event_cleanup(void);
2013-06-26 11:18:32 +02:00
void kore_platform_proctitle(char *);
void kore_platform_disable_read(int);
void kore_platform_disable_write(int);
void kore_platform_enable_accept(void);
void kore_platform_disable_accept(void);
void kore_platform_event_wait(u_int64_t);
void kore_platform_event_all(int, void *);
void kore_platform_schedule_read(int, void *);
2015-12-09 21:29:44 +01:00
void kore_platform_schedule_write(int, void *);
2013-06-26 11:18:32 +02:00
void kore_platform_event_schedule(int, int, int, void *);
void kore_platform_worker_setcpu(struct kore_worker *);
#if defined(KORE_USE_PLATFORM_SENDFILE)
int kore_platform_sendfile(struct connection *, struct netbuf *);
#endif
#if defined(KORE_USE_PLATFORM_PLEDGE)
void kore_platform_pledge(void);
void kore_platform_add_pledge(const char *);
#endif
void kore_accesslog_init(u_int16_t);
2013-06-24 09:36:40 +02:00
void kore_accesslog_worker_init(void);
void kore_accesslog_run(void *, u_int64_t);
void kore_accesslog_gather(void *, u_int64_t, int);
#if !defined(KORE_NO_HTTP)
2015-04-02 13:45:42 +02:00
int kore_auth_run(struct http_request *, struct kore_auth *);
2018-07-19 10:28:38 +02:00
int kore_auth_cookie(struct http_request *, struct kore_auth *);
int kore_auth_header(struct http_request *, struct kore_auth *);
int kore_auth_request(struct http_request *, struct kore_auth *);
2015-04-02 13:45:42 +02:00
void kore_auth_init(void);
int kore_auth_new(const char *);
2014-08-04 12:40:21 +02:00
struct kore_auth *kore_auth_lookup(const char *);
#endif
void kore_timer_init(void);
void kore_timer_run(u_int64_t);
u_int64_t kore_timer_next_run(u_int64_t);
void kore_timer_remove(struct kore_timer *);
struct kore_timer *kore_timer_add(void (*cb)(void *, u_int64_t),
u_int64_t, void *, int);
void kore_listener_cleanup(void);
void kore_listener_accept(void *, int);
void kore_listener_free(struct listener *);
struct listener *kore_listener_alloc(int, const char *);
int kore_sockopt(int, int, int);
int kore_server_bind_unix(const char *, const char *);
int kore_server_bind(const char *, const char *, const char *);
#if !defined(KORE_NO_TLS)
int kore_tls_sni_cb(SSL *, int *, void *);
void kore_tls_info_callback(const SSL *, int, int);
#endif
2013-05-01 16:03:48 +02:00
void kore_connection_init(void);
2016-01-04 22:40:14 +01:00
void kore_connection_cleanup(void);
void kore_connection_prune(int);
struct connection *kore_connection_new(void *);
void kore_connection_event(void *, int);
int kore_connection_nonblock(int, int);
Rework HTTP and worker processes. The HTTP layer used to make a copy of each incoming header and its value for a request. Stop doing that and make HTTP headers zero-copy all across the board. This change comes with some api function changes, notably the http_request_header() function which now takes a const char ** rather than a char ** out pointer. This commit also constifies several members of http_request, beware. Additional rework how the worker processes deal with the accept lock. Before: if a worker held the accept lock and it accepted a new connection it would release the lock for others and back off for 500ms before attempting to grab the lock again. This approach worked but under high load this starts becoming obvious. Now: - workers not holding the accept lock and not having any connections will wait less long before returning from kore_platform_event_wait(). - workers not holding the accept lock will no longer blindly wait an arbitrary amount in kore_platform_event_wait() but will look at how long until the next lock grab is and base their timeout on that. - if a worker its next_lock timeout is up and failed to grab the lock it will try again in half the time again. - the worker process holding the lock will when releasing the lock double check if it still has space for newer connections, if it does it will keep the lock until it is full. This prevents the lock from bouncing between several non busy worker processes all the time. Additional fixes: - Reduce the number of times we check the timeout list, only do it twice per second rather then every event tick. - Fix solo worker count for TLS (we actually hold two processes, not one). - Make sure we don't accidentally miscalculate the idle time causing new connections under heavy load to instantly drop. - Swap from gettimeofday() to clock_gettime() now that MacOS caught up.
2018-02-14 13:48:49 +01:00
void kore_connection_check_timeout(u_int64_t);
int kore_connection_handle(struct connection *);
void kore_connection_remove(struct connection *);
void kore_connection_disconnect(struct connection *);
void kore_connection_start_idletimer(struct connection *);
void kore_connection_stop_idletimer(struct connection *);
void kore_connection_check_idletimer(u_int64_t,
struct connection *);
int kore_connection_accept(struct listener *,
struct connection **);
2013-06-26 11:18:32 +02:00
2013-06-24 09:36:40 +02:00
u_int64_t kore_time_ms(void);
void kore_log_init(void);
2013-04-17 22:34:27 +02:00
void *kore_malloc(size_t);
2013-06-26 11:18:32 +02:00
void kore_parse_config(void);
void kore_parse_config_file(FILE *);
2013-04-17 22:34:27 +02:00
void *kore_calloc(size_t, size_t);
void *kore_realloc(void *, size_t);
void kore_free(void *);
void kore_mem_init(void);
void kore_mem_cleanup(void);
void kore_mem_untag(void *);
void *kore_mem_lookup(u_int32_t);
void kore_mem_tag(void *, u_int32_t);
void *kore_malloc_tagged(size_t, u_int32_t);
void *kore_pool_get(struct kore_pool *);
void kore_pool_put(struct kore_pool *, void *);
2014-08-04 12:40:21 +02:00
void kore_pool_init(struct kore_pool *, const char *,
2016-07-12 14:01:02 +02:00
size_t, size_t);
2016-01-04 22:40:14 +01:00
void kore_pool_cleanup(struct kore_pool *);
char *kore_time_to_date(time_t);
2013-04-17 22:34:27 +02:00
char *kore_strdup(const char *);
time_t kore_date_to_time(const char *);
void kore_log(int, const char *, ...)
__attribute__((format (printf, 2, 3)));
u_int64_t kore_strtonum64(const char *, int, int *);
size_t kore_strlcpy(char *, const char *, const size_t);
void kore_server_disconnect(struct connection *);
2016-10-06 16:50:41 +02:00
int kore_split_string(char *, const char *, char **, size_t);
2016-02-13 14:33:06 +01:00
void kore_strip_chars(char *, const char, char **);
int kore_snprintf(char *, size_t, int *, const char *, ...);
long long kore_strtonum(const char *, int, long long, long long, int *);
double kore_strtodouble(const char *, long double, long double, int *);
2017-07-24 08:19:03 +02:00
int kore_base64_encode(const void *, size_t, char **);
int kore_base64_decode(const char *, u_int8_t **, size_t *);
2018-07-25 09:54:34 +02:00
void *kore_mem_find(void *, size_t, const void *, size_t);
char *kore_text_trim(char *, size_t);
char *kore_read_line(FILE *, char *, size_t);
2013-05-01 16:03:48 +02:00
#if !defined(KORE_NO_HTTP)
void kore_websocket_handshake(struct http_request *,
const char *, const char *, const char *);
int kore_websocket_send_clean(struct netbuf *);
void kore_websocket_send(struct connection *,
u_int8_t, const void *, size_t);
void kore_websocket_broadcast(struct connection *,
u_int8_t, const void *, size_t, int);
#endif
void kore_msg_init(void);
void kore_msg_worker_init(void);
void kore_msg_parent_init(void);
void kore_msg_parent_add(struct kore_worker *);
void kore_msg_parent_remove(struct kore_worker *);
void kore_msg_send(u_int16_t, u_int8_t, const void *, size_t);
int kore_msg_register(u_int8_t,
void (*cb)(struct kore_msg *, const void *));
#if !defined(KORE_NO_HTTP)
void kore_filemap_init(void);
void kore_filemap_resolve_paths(void);
int kore_filemap_create(struct kore_domain *, const char *,
const char *);
extern char *kore_filemap_ext;
extern char *kore_filemap_index;
#endif
void kore_fileref_init(void);
struct kore_fileref *kore_fileref_get(const char *);
struct kore_fileref *kore_fileref_create(const char *, int, off_t,
struct timespec *);
void kore_fileref_release(struct kore_fileref *);
void kore_domain_init(void);
void kore_domain_cleanup(void);
int kore_domain_new(char *);
void kore_domain_free(struct kore_domain *);
void kore_module_init(void);
void kore_module_cleanup(void);
void kore_module_reload(int);
void kore_module_onload(void);
2013-05-01 16:03:48 +02:00
int kore_module_loaded(void);
void kore_domain_closelogs(void);
void *kore_module_getsym(const char *, struct kore_runtime **);
void kore_domain_load_crl(void);
void kore_domain_keymgr_init(void);
void kore_domain_callback(void (*cb)(struct kore_domain *));
void kore_domain_tlsinit(struct kore_domain *, const void *, size_t);
void kore_domain_crl_add(struct kore_domain *, const void *, size_t);
#if !defined(KORE_NO_HTTP)
2014-08-04 12:40:21 +02:00
int kore_module_handler_new(const char *, const char *,
const char *, const char *, int);
void kore_module_handler_free(struct kore_module_handle *);
struct kore_module_handle *kore_module_handler_find(const char *,
const char *);
#endif
2014-08-04 12:40:21 +02:00
struct kore_runtime_call *kore_runtime_getcall(const char *);
struct kore_module *kore_module_load(const char *,
const char *, int);
void kore_runtime_execute(struct kore_runtime_call *);
int kore_runtime_onload(struct kore_runtime_call *, int);
void kore_runtime_configure(struct kore_runtime_call *, int, char **);
void kore_runtime_connect(struct kore_runtime_call *, struct connection *);
#if !defined(KORE_NO_HTTP)
int kore_runtime_http_request(struct kore_runtime_call *,
struct http_request *);
int kore_runtime_validator(struct kore_runtime_call *,
Rework HTTP and worker processes. The HTTP layer used to make a copy of each incoming header and its value for a request. Stop doing that and make HTTP headers zero-copy all across the board. This change comes with some api function changes, notably the http_request_header() function which now takes a const char ** rather than a char ** out pointer. This commit also constifies several members of http_request, beware. Additional rework how the worker processes deal with the accept lock. Before: if a worker held the accept lock and it accepted a new connection it would release the lock for others and back off for 500ms before attempting to grab the lock again. This approach worked but under high load this starts becoming obvious. Now: - workers not holding the accept lock and not having any connections will wait less long before returning from kore_platform_event_wait(). - workers not holding the accept lock will no longer blindly wait an arbitrary amount in kore_platform_event_wait() but will look at how long until the next lock grab is and base their timeout on that. - if a worker its next_lock timeout is up and failed to grab the lock it will try again in half the time again. - the worker process holding the lock will when releasing the lock double check if it still has space for newer connections, if it does it will keep the lock until it is full. This prevents the lock from bouncing between several non busy worker processes all the time. Additional fixes: - Reduce the number of times we check the timeout list, only do it twice per second rather then every event tick. - Fix solo worker count for TLS (we actually hold two processes, not one). - Make sure we don't accidentally miscalculate the idle time causing new connections under heavy load to instantly drop. - Swap from gettimeofday() to clock_gettime() now that MacOS caught up.
2018-02-14 13:48:49 +01:00
struct http_request *, const void *);
void kore_runtime_wsconnect(struct kore_runtime_call *, struct connection *);
void kore_runtime_wsdisconnect(struct kore_runtime_call *,
struct connection *);
void kore_runtime_wsmessage(struct kore_runtime_call *,
struct connection *, u_int8_t, const void *, size_t);
#endif
struct kore_domain *kore_domain_byid(u_int16_t);
struct kore_domain *kore_domain_lookup(const char *);
2013-04-17 22:34:27 +02:00
#if !defined(KORE_NO_HTTP)
void kore_validator_init(void);
void kore_validator_reload(void);
2014-08-04 12:40:21 +02:00
int kore_validator_add(const char *, u_int8_t, const char *);
int kore_validator_run(struct http_request *, const char *, char *);
int kore_validator_check(struct http_request *,
Rework HTTP and worker processes. The HTTP layer used to make a copy of each incoming header and its value for a request. Stop doing that and make HTTP headers zero-copy all across the board. This change comes with some api function changes, notably the http_request_header() function which now takes a const char ** rather than a char ** out pointer. This commit also constifies several members of http_request, beware. Additional rework how the worker processes deal with the accept lock. Before: if a worker held the accept lock and it accepted a new connection it would release the lock for others and back off for 500ms before attempting to grab the lock again. This approach worked but under high load this starts becoming obvious. Now: - workers not holding the accept lock and not having any connections will wait less long before returning from kore_platform_event_wait(). - workers not holding the accept lock will no longer blindly wait an arbitrary amount in kore_platform_event_wait() but will look at how long until the next lock grab is and base their timeout on that. - if a worker its next_lock timeout is up and failed to grab the lock it will try again in half the time again. - the worker process holding the lock will when releasing the lock double check if it still has space for newer connections, if it does it will keep the lock until it is full. This prevents the lock from bouncing between several non busy worker processes all the time. Additional fixes: - Reduce the number of times we check the timeout list, only do it twice per second rather then every event tick. - Fix solo worker count for TLS (we actually hold two processes, not one). - Make sure we don't accidentally miscalculate the idle time causing new connections under heavy load to instantly drop. - Swap from gettimeofday() to clock_gettime() now that MacOS caught up.
2018-02-14 13:48:49 +01:00
struct kore_validator *, const void *);
2014-08-04 12:40:21 +02:00
struct kore_validator *kore_validator_lookup(const char *);
#endif
void fatal(const char *, ...) __attribute__((noreturn));
void fatalx(const char *, ...) __attribute__((noreturn));
void kore_debug_internal(char *, int, const char *, ...);
2013-04-17 22:34:27 +02:00
u_int16_t net_read16(u_int8_t *);
u_int32_t net_read32(u_int8_t *);
u_int64_t net_read64(u_int8_t *);
void net_write16(u_int8_t *, u_int16_t);
void net_write32(u_int8_t *, u_int32_t);
void net_write64(u_int8_t *, u_int64_t);
void net_init(void);
2016-01-04 22:40:14 +01:00
void net_cleanup(void);
struct netbuf *net_netbuf_get(void);
int net_send(struct connection *);
int net_send_flush(struct connection *);
int net_recv_flush(struct connection *);
int net_read(struct connection *, size_t *);
int net_read_tls(struct connection *, size_t *);
int net_write(struct connection *, size_t, size_t *);
int net_write_tls(struct connection *, size_t, size_t *);
void net_recv_reset(struct connection *, size_t,
int (*cb)(struct netbuf *));
void net_remove_netbuf(struct connection *, struct netbuf *);
void net_recv_queue(struct connection *, size_t, int,
int (*cb)(struct netbuf *));
void net_recv_expand(struct connection *c, size_t,
int (*cb)(struct netbuf *));
void net_send_queue(struct connection *, const void *, size_t);
void net_send_stream(struct connection *, void *,
size_t, int (*cb)(struct netbuf *), struct netbuf **);
void net_send_fileref(struct connection *, struct kore_fileref *);
void kore_buf_free(struct kore_buf *);
struct kore_buf *kore_buf_alloc(size_t);
void kore_buf_init(struct kore_buf *, size_t);
void kore_buf_append(struct kore_buf *, const void *, size_t);
u_int8_t *kore_buf_release(struct kore_buf *, size_t *);
Massive rework of HTTP layer. This commit is a flag day, your old modules will almost certainly need to be updated in order to build properly with these changes. Summary of changes: - Offload HTTP bodies to disk if they are large (inspired by #100). (disabled by default) - The http_argument_get* macros now takes an explicit http_request parameter. - Kore will now throw 404 errors almost immediately after an HTTP request has come in instead of waiting until all data has arrived. API changes: - http_argument_get* macros now require an explicit http_request parameter. (no more magic invokations). - http_generic_404() is gone - http_populate_arguments() is gone - http_body_bytes() is gone - http_body_text() is gone - http_body_read() has been added - http_populate_post() has been added - http_populate_get() has been added - http_file_read() has been added - http_file_rewind() has been added - http_file_lookup() no longer takes name, fname, data and len parameters. - http_file_lookup() now returns a struct http_file pointer. - http_populate_multipart_form() no longer takes an secondary parameter. New configuration options: - http_body_disk_offload: Number of bytes after which Kore will offload the HTTP body to disk instead of retaining it in memory. If 0 this feature is disabled. (Default: 0) - http_body_disk_path: The path where Kore will store temporary HTTP body files. (this directory does not get created if http_body_disk_offload is 0). New example: The upload example has been added, demonstrating how to deal with file uploads from a multipart form.
2016-01-18 11:30:22 +01:00
void kore_buf_reset(struct kore_buf *);
void kore_buf_cleanup(struct kore_buf *);
char *kore_buf_stringify(struct kore_buf *, size_t *);
void kore_buf_appendf(struct kore_buf *, const char *, ...);
void kore_buf_appendv(struct kore_buf *, const char *, va_list);
2018-07-25 09:54:34 +02:00
void kore_buf_replace_string(struct kore_buf *,
const char *, const void *, size_t);
void kore_keymgr_run(void);
void kore_keymgr_cleanup(int);
void kore_seccomp_hook(void);
void kore_worker_teardown(void);
void kore_parent_teardown(void);
void kore_worker_configure(void);
void kore_parent_daemonized(void);
void kore_parent_configure(int, char **);
2015-04-01 13:25:10 +02:00
#if defined(__cplusplus)
}
#endif
2013-04-17 22:34:27 +02:00
#endif /* !__H_KORE_H */