a1batross
/
kore
forked from mirrors/kore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improvements to our message framework. 

Change the callback prototypes to:
	void callback(struct kore_msg *msg, const void *data);

This allows the callbacks to receive the full kore_msg data structure
as sent over the wire (including length and id). Useful for future
additions to the kore_msg structure (such as worker origin).

Several other improvements:
	* Accesslog now uses the msg framework as well.
	* Websocket WEBSOCKET_BROADCAST_GLOBAL now works.

Small websocket improvement in this commit:
	* Build the frame to be sent only once when broadcasting
	 instead of per connection we are broadcasting towards.
This commit is contained in:
Joris Vink 2015-06-23 18:17:14 +02:00
parent 0e3271cf9d
commit a64808c6b0
10 changed files with 121 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
examples/messaging/src/messaging.c
View File

@ -28,7 +28,7 @@
int init(int);
int page(struct http_request *);
void received_message(const void *, u_int32_t);
void received_message(struct kore_msg *, const void *);
/* Initialization callback. */
int
@ -51,10 +51,10 @@ init(int state)
* Callback for receiving a message MY_MESSAGE_ID.
*/
void
received_message(const void *data, u_int32_t len)
received_message(struct kore_msg *msg, const void *data)
{
kore_log(LOG_INFO, "got message (%d bytes): %.*s", len,
len, (const char *)data);
kore_log(LOG_INFO, "got message (%d bytes): %.*s", msg->length,
msg->length, (const char *)data);
}
/*

4
examples/websocket/conf/websocket.conf
View File

@ -5,6 +5,10 @@ load ./websocket.so
tls_dhparam dh2048.pem
# Increase workers so connections are spread
# across them to demonstrate WEBSOCKET_BROADCAST_GLOBAL.
workers 4
websocket_maxframe 65536
websocket_timeout 20

2
examples/websocket/src/websocket.c
View File

@ -44,7 +44,7 @@ websocket_connect(struct connection *c)
void
websocket_message(struct connection *c, u_int8_t op, void *data, size_t len)
{
kore_websocket_broadcast(c, op, data, len, WEBSOCKET_BROADCAST_LOCAL);
kore_websocket_broadcast(c, op, data, len, WEBSOCKET_BROADCAST_GLOBAL);
}
void

17
includes/kore.h
View File

@ -365,6 +365,10 @@ struct kore_timer {
TAILQ_ENTRY(kore_timer) list;
};
/* Reserved message ids, registered on workers. */
#define KORE_MSG_ACCESSLOG 1
#define KORE_MSG_WEBSOCKET 2
struct kore_msg {
u_int8_t id;
u_int32_t length;
@ -428,8 +432,8 @@ void kore_platform_event_schedule(int, int, int, void *);
void kore_platform_worker_setcpu(struct kore_worker *);
void kore_accesslog_init(void);
int kore_accesslog_wait(void);
void kore_accesslog_worker_init(void);
int kore_accesslog_write(const void *, u_int32_t);
int kore_auth_run(struct http_request *, struct kore_auth *);
void kore_auth_init(void);
@ -499,9 +503,9 @@ void *kore_mem_find(void *, size_t, void *, u_int32_t);
void kore_websocket_handshake(struct http_request *,
struct kore_wscbs *);
void kore_websocket_send(struct connection *,
u_int8_t, void *, size_t);
u_int8_t, const void *, size_t);
void kore_websocket_broadcast(struct connection *,
u_int8_t, void *, size_t, int);
u_int8_t, const void *, size_t, int);
void kore_msg_init(void);
void kore_msg_worker_init(void);
@ -510,7 +514,7 @@ void kore_msg_parent_add(struct kore_worker *);
void kore_msg_parent_remove(struct kore_worker *);
void kore_msg_send(u_int8_t, void *, u_int32_t);
int kore_msg_register(u_int8_t,
void (*cb)(const void *, u_int32_t));
void (*cb)(struct kore_msg *, const void *));
void kore_domain_init(void);
int kore_domain_new(char *);
@ -563,7 +567,7 @@ void net_recv_queue(struct connection *, u_int32_t, int,
int (*cb)(struct netbuf *));
void net_recv_expand(struct connection *c, u_int32_t,
int (*cb)(struct netbuf *));
void net_send_queue(struct connection *, void *,
void net_send_queue(struct connection *, const void *,
u_int32_t, struct spdy_stream *, int);
void net_send_stream(struct connection *, void *,
u_int32_t, struct spdy_stream *,
@ -571,8 +575,9 @@ void net_send_stream(struct connection *, void *,
void kore_buf_free(struct kore_buf *);
struct kore_buf *kore_buf_create(u_int32_t);
void kore_buf_append(struct kore_buf *, void *, u_int32_t);
void kore_buf_append(struct kore_buf *, const void *, u_int32_t);
u_int8_t *kore_buf_release(struct kore_buf *, u_int32_t *);
void kore_buf_appendf(struct kore_buf *, const char *, ...);
void kore_buf_appendv(struct kore_buf *, const char *, va_list);
void kore_buf_appendb(struct kore_buf *, struct kore_buf *);

53
src/accesslog.c
View File

@ -21,8 +21,6 @@
#include "kore.h"
#include "http.h"
static int accesslog_fd[2];
struct kore_log_packet {
u_int8_t method;
int status;
@ -40,52 +38,29 @@ struct kore_log_packet {
void
kore_accesslog_init(void)
{
if (socketpair(AF_UNIX, SOCK_STREAM, 0, accesslog_fd) == -1)
fatal("kore_accesslog_init(): socketpair() %s", errno_s);
}
void
kore_accesslog_worker_init(void)
{
close(accesslog_fd[0]);
kore_domain_closelogs();
}
int
kore_accesslog_wait(void)
kore_accesslog_write(const void *data, u_int32_t len)
{
ssize_t len;
int l;
time_t now;
ssize_t sent;
struct kore_domain *dom;
struct pollfd pfd[1];
int nfds, l;
struct kore_log_packet logpacket;
char addr[INET6_ADDRSTRLEN];
char *method, *buf, *tbuf, *cn;
pfd[0].fd = accesslog_fd[0];
pfd[0].events = POLLIN;
pfd[0].revents = 0;
nfds = poll(pfd, 1, 100);
if (nfds == -1 || (pfd[0].revents & (POLLERR | POLLHUP | POLLNVAL))) {
if (nfds == -1 && errno == EINTR)
return (KORE_RESULT_OK);
kore_log(LOG_WARNING, "poll(): %s", errno_s);
if (len != sizeof(struct kore_log_packet))
return (KORE_RESULT_ERROR);
}
if (nfds == 0)
return (KORE_RESULT_OK);
len = recv(accesslog_fd[0], &logpacket, sizeof(logpacket), 0);
if (len == -1) {
kore_log(LOG_WARNING, "recv(): %s", errno_s);
return (KORE_RESULT_ERROR);
}
if (len != sizeof(logpacket))
return (KORE_RESULT_ERROR);
(void)memcpy(&logpacket, data, sizeof(logpacket));
if ((dom = kore_domain_lookup(logpacket.host)) == NULL) {
kore_log(LOG_WARNING,
@ -131,19 +106,19 @@ kore_accesslog_wait(void)
logpacket.worker_id, logpacket.time_req, cn, logpacket.agent);
if (l == -1) {
kore_log(LOG_WARNING,
"kore_accesslog_wait(): asprintf() == -1");
"kore_accesslog_write(): asprintf() == -1");
return (KORE_RESULT_ERROR);
}
len = write(dom->accesslog, buf, l);
if (len == -1) {
sent = write(dom->accesslog, buf, l);
if (sent == -1) {
free(buf);
kore_log(LOG_WARNING,
"kore_accesslog_wait(): write(): %s", errno_s);
"kore_accesslog_write(): write(): %s", errno_s);
return (KORE_RESULT_ERROR);
}
if (len != l)
if (sent != l)
kore_log(LOG_NOTICE, "accesslog: %s", buf);
free(buf);
@ -153,7 +128,6 @@ kore_accesslog_wait(void)
void
kore_accesslog(struct http_request *req)
{
ssize_t len;
struct kore_log_packet logpacket;
logpacket.addrtype = req->owner->addrtype;
@ -193,10 +167,5 @@ kore_accesslog(struct http_request *req)
}
#endif
len = send(accesslog_fd[1], &logpacket, sizeof(logpacket), 0);
if (len == -1) {
kore_log(LOG_WARNING, "kore_accesslog(): send(): %s", errno_s);
} else if (len != sizeof(logpacket)) {
kore_log(LOG_WARNING, "short accesslog packet sent");
}
kore_msg_send(KORE_MSG_ACCESSLOG, &logpacket, sizeof(logpacket));
}

2
src/buf.c
View File

@ -30,7 +30,7 @@ kore_buf_create(u_int32_t initial)
}
void
kore_buf_append(struct kore_buf *buf, void *d, u_int32_t len)
kore_buf_append(struct kore_buf *buf, const void *d, u_int32_t len)
{
if ((buf->offset + len) >= buf->length) {
buf->length += len + KORE_BUF_INCREMENT;

6
src/kore.c
View File

@ -383,12 +383,6 @@ kore_server_start(void)
sig_recv = 0;
}
/* XXX - The accesslog should move to our msg framework. */
if (!kore_accesslog_wait()) {
kore_worker_dispatch_signal(SIGQUIT);
break;
}
kore_worker_wait(0);
kore_platform_event_wait(100);
kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT);

75
src/msg.c
View File

@ -23,17 +23,18 @@
struct msg_type {
u_int8_t id;
void (*cb)(const void *, u_int32_t);
void (*cb)(struct kore_msg *, const void *);
TAILQ_ENTRY(msg_type) list;
};
TAILQ_HEAD(, msg_type) msg_types;
static struct msg_type *msg_type_lookup(u_int8_t);
static int msg_recv_worker(struct netbuf *);
static int msg_recv_parent(struct netbuf *);
static int msg_recv_worker_data(struct netbuf *);
static int msg_recv_packet(struct netbuf *);
static int msg_recv_data(struct netbuf *);
static void msg_disconnected_parent(struct connection *);
static void msg_type_accesslog(struct kore_msg *, const void *);
static void msg_type_websocket(struct kore_msg *, const void *);
void
kore_msg_init(void)
@ -51,6 +52,8 @@ kore_msg_parent_init(void)
kw = kore_worker_data(i);
kore_msg_parent_add(kw);
}
kore_msg_register(KORE_MSG_ACCESSLOG, msg_type_accesslog);
}
void
@ -66,8 +69,7 @@ kore_msg_parent_add(struct kore_worker *kw)
TAILQ_INSERT_TAIL(&connections, kw->msg[0], list);
kore_platform_event_all(kw->msg[0]->fd, kw->msg[0]);
net_recv_queue(kw->msg[0], NETBUF_SEND_PAYLOAD_MAX,
NETBUF_CALL_CB_ALWAYS, msg_recv_parent);
net_recv_queue(kw->msg[0], sizeof(struct kore_msg), 0, msg_recv_packet);
}
void
@ -81,6 +83,8 @@ kore_msg_parent_remove(struct kore_worker *kw)
void
kore_msg_worker_init(void)
{
kore_msg_register(KORE_MSG_WEBSOCKET, msg_type_websocket);
worker->msg[1] = kore_connection_new(NULL);
worker->msg[1]->fd = worker->pipe[1];
worker->msg[1]->read = net_read;
@ -93,11 +97,11 @@ kore_msg_worker_init(void)
kore_platform_event_all(worker->msg[1]->fd, worker->msg[1]);
net_recv_queue(worker->msg[1],
sizeof(struct kore_msg), 0, msg_recv_worker);
sizeof(struct kore_msg), 0, msg_recv_packet);
}
int
kore_msg_register(u_int8_t id, void (*cb)(const void *, u_int32_t))
kore_msg_register(u_int8_t id, void (*cb)(struct kore_msg *, const void *))
{
struct msg_type *type;
@ -126,41 +130,35 @@ kore_msg_send(u_int8_t id, void *data, u_int32_t len)
}
static int
msg_recv_worker(struct netbuf *nb)
msg_recv_packet(struct netbuf *nb)
{
struct kore_msg *msg = (struct kore_msg *)nb->buf;
net_recv_expand(nb->owner, msg->length, msg_recv_worker_data);
net_recv_expand(nb->owner, msg->length, msg_recv_data);
return (KORE_RESULT_OK);
}
static int
msg_recv_worker_data(struct netbuf *nb)
msg_recv_data(struct netbuf *nb)
{
struct connection *c;
struct msg_type *type;
struct kore_msg *msg = (struct kore_msg *)nb->buf;
if ((type = msg_type_lookup(msg->id)) != NULL)
type->cb(nb->buf + sizeof(*msg), nb->s_off - sizeof(*msg));
type->cb(msg, nb->buf + sizeof(*msg));
net_recv_reset(nb->owner, sizeof(struct kore_msg), msg_recv_worker);
return (KORE_RESULT_OK);
}
static int
msg_recv_parent(struct netbuf *nb)
{
struct connection *c;
TAILQ_FOREACH(c, &connections, list) {
if (c == nb->owner)
continue;
net_send_queue(c, nb->buf, nb->s_off, NULL, NETBUF_LAST_CHAIN);
net_send_flush(c);
if (worker == NULL && type == NULL) {
TAILQ_FOREACH(c, &connections, list) {
if (c == nb->owner)
continue;
net_send_queue(c, nb->buf, nb->s_off,
NULL, NETBUF_LAST_CHAIN);
net_send_flush(c);
}
}
net_recv_reset(nb->owner, NETBUF_SEND_PAYLOAD_MAX, msg_recv_parent);
net_recv_reset(nb->owner, sizeof(struct kore_msg), msg_recv_packet);
return (KORE_RESULT_OK);
}
@ -172,6 +170,27 @@ msg_disconnected_parent(struct connection *c)
kore_log(LOG_ERR, "failed to send SIGQUIT: %s", errno_s);
}
static void
msg_type_accesslog(struct kore_msg *msg, const void *data)
{
if (kore_accesslog_write(data, msg->length) == -1)
kore_log(LOG_WARNING, "failed to write to accesslog");
}
static void
msg_type_websocket(struct kore_msg *msg, const void *data)
{
struct connection *c;
TAILQ_FOREACH(c, &connections, list) {
if (c->proto == CONN_PROTO_WEBSOCKET) {
net_send_queue(c, data, msg->length,
NULL, NETBUF_LAST_CHAIN);
net_send_flush(c);
}
}
}
static struct msg_type *
msg_type_lookup(u_int8_t id)
{

4
src/net.c
View File

@ -37,10 +37,10 @@ net_init(void)
}
void
net_send_queue(struct connection *c, void *data, u_int32_t len,
net_send_queue(struct connection *c, const void *data, u_int32_t len,
struct spdy_stream *s, int before)
{
u_int8_t *d;
const u_int8_t *d;
struct netbuf *nb;
u_int32_t avail;

76
src/websocket.c
View File

@ -42,9 +42,8 @@ u_int64_t kore_websocket_maxframe = 16384;
static int websocket_recv_frame(struct netbuf *);
static int websocket_recv_opcode(struct netbuf *);
static void websocket_disconnect(struct connection *);
static void websocket_send_single(struct connection *,
u_int8_t, void *, size_t);
void websocket_send(struct connection *, u_int8_t, void *, size_t);
static void websocket_frame_build(struct kore_buf *, u_int8_t,
const void *, size_t);
void
kore_websocket_handshake(struct http_request *req, struct kore_wscbs *wscbs)
@ -115,17 +114,48 @@ kore_websocket_handshake(struct http_request *req, struct kore_wscbs *wscbs)
}
void
kore_websocket_send(struct connection *c, u_int8_t op, void *data, size_t len)
kore_websocket_send(struct connection *c, u_int8_t op, const void *data,
size_t len)
{
struct kore_buf *frame;
frame = kore_buf_create(len);
websocket_frame_build(frame, op, data, len);
net_send_queue(c, frame->data, frame->offset, NULL, NETBUF_LAST_CHAIN);
kore_buf_free(frame);
}
void
kore_websocket_broadcast(struct connection *src, u_int8_t op, const void *data,
size_t len, int scope)
{
struct connection *c;
struct kore_buf *frame;
frame = kore_buf_create(len);
websocket_frame_build(frame, op, data, len);
TAILQ_FOREACH(c, &connections, list) {
if (c != src && c->proto == CONN_PROTO_WEBSOCKET) {
net_send_queue(c, frame->data, frame->offset,
NULL, NETBUF_LAST_CHAIN);
net_send_flush(c);
}
}
if (scope == WEBSOCKET_BROADCAST_GLOBAL)
kore_msg_send(KORE_MSG_WEBSOCKET, frame->data, frame->offset);
kore_buf_free(frame);
}
static void
websocket_frame_build(struct kore_buf *frame, u_int8_t op, const void *data,
size_t len)
{
u_int8_t len_1;
u_int16_t len16;
u_int64_t len64;
struct kore_buf *frame;
if (c->proto != CONN_PROTO_WEBSOCKET)
fatal("kore_websocket_send(): to non websocket connection");
kore_debug("%p: sending %ld bytes", c, len);
if (len > WEBSOCKET_PAYLOAD_SINGLE) {
if (len < USHRT_MAX)
@ -136,8 +166,6 @@ kore_websocket_send(struct connection *c, u_int8_t op, void *data, size_t len)
len_1 = len;
}
frame = kore_buf_create(len);
op |= (1 << 7);
kore_buf_append(frame, &op, sizeof(op));
@ -158,30 +186,6 @@ kore_websocket_send(struct connection *c, u_int8_t op, void *data, size_t len)
}
kore_buf_append(frame, data, len);
net_send_queue(c, frame->data, frame->offset, NULL, NETBUF_LAST_CHAIN);
kore_buf_free(frame);
}
void
kore_websocket_broadcast(struct connection *src, u_int8_t op, void *data,
size_t len, int scope)
{
struct connection *c;
TAILQ_FOREACH(c, &connections, list) {
if (c != src && c->proto == CONN_PROTO_WEBSOCKET)
websocket_send_single(c, op, data, len);
}
if (scope == WEBSOCKET_BROADCAST_GLOBAL)
fatal("kore_websocket_broadcast: no global scope yet");
}
static void
websocket_send_single(struct connection *c, u_int8_t op, void *data, size_t len)
{
kore_websocket_send(c, op, data, len);
net_send_flush(c);
}
static int