diff --git a/src/Makefile b/src/Makefile index 1ba681fdb7d..949490fa18b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -266,7 +266,8 @@ RUNTIME_CS := rt/sync/timer.cpp \ rt/rust_timer.cpp \ rt/circular_buffer.cpp \ rt/isaac/randport.cpp \ - rt/rust_srv.cpp + rt/rust_srv.cpp \ + rt/memory_region.cpp RUNTIME_HDR := rt/globals.h \ rt/rust.h \ @@ -285,7 +286,9 @@ RUNTIME_HDR := rt/globals.h \ rt/util/hash_map.h \ rt/sync/sync.h \ rt/sync/timer.h \ - rt/rust_srv.h + rt/rust_srv.h \ + rt/memory_region.h \ + rt/memory.h RUNTIME_INCS := -Irt/isaac -Irt/uthash RUNTIME_OBJS := $(RUNTIME_CS:.cpp=$(CFG_OBJ_SUFFIX)) diff --git a/src/rt/memory.h b/src/rt/memory.h new file mode 100644 index 00000000000..22bc15d32b6 --- /dev/null +++ b/src/rt/memory.h @@ -0,0 +1,60 @@ +/* + * + */ + +#ifndef MEMORY_H +#define MEMORY_H + + +inline void *operator new(size_t size, void *mem) { + return mem; +} + +inline void *operator new(size_t size, rust_dom *dom) { + return dom->malloc(size, memory_region::LOCAL); +} + +inline void *operator new[](size_t size, rust_dom *dom) { + return dom->malloc(size, memory_region::LOCAL); +} + +inline void *operator new(size_t size, rust_dom &dom) { + return dom.malloc(size, memory_region::LOCAL); +} + +inline void *operator new[](size_t size, rust_dom &dom) { + return dom.malloc(size, memory_region::LOCAL); +} + +inline void *operator new(size_t size, rust_dom *dom, + memory_region::memory_region_type type) { + return dom->malloc(size, type); +} + +inline void *operator new[](size_t size, rust_dom *dom, + memory_region::memory_region_type type) { + return dom->malloc(size, type); +} + +inline void *operator new(size_t size, rust_dom &dom, + memory_region::memory_region_type type) { + return dom.malloc(size, type); +} + +inline void *operator new[](size_t size, rust_dom &dom, + memory_region::memory_region_type type) { + return dom.malloc(size, type); +} + +inline void operator delete(void *mem, rust_dom *dom) { + dom->free(mem, memory_region::LOCAL); + return; +} + +inline void operator delete(void *mem, rust_dom *dom, + memory_region::memory_region_type type) { + dom->free(mem, type); + return; +} + +#endif /* MEMORY_H */ diff --git a/src/rt/memory_region.cpp b/src/rt/memory_region.cpp new file mode 100644 index 00000000000..797a7c1d669 --- /dev/null +++ b/src/rt/memory_region.cpp @@ -0,0 +1,100 @@ +/* + * + */ + +#include "rust_internal.h" +#include "memory_region.h" + +#define TRACK_ALLOCATIONS + +memory_region::memory_region(rust_srv *srv, bool synchronized) : + _srv(srv), _parent(NULL), _live_allocations(0), + _synchronized(synchronized) { + // Nop. +} + +memory_region::memory_region(memory_region *parent) : + _srv(parent->_srv), _parent(parent), _live_allocations(0), + _synchronized(parent->_synchronized) { + // Nop. +} + +void memory_region::free(void *mem) { + if (_synchronized) { _lock.lock(); } +#ifdef TRACK_ALLOCATIONS + if (_allocation_list.replace(mem, NULL) == false) { + printf("free: ptr 0x%" PRIxPTR " is not in allocation_list\n", + (uintptr_t) mem); + _srv->fatal("not in allocation_list", __FILE__, __LINE__, ""); + } +#endif + if (_live_allocations < 1) { + _srv->fatal("live_allocs < 1", __FILE__, __LINE__, ""); + } + _live_allocations--; + _srv->free(mem); + if (_synchronized) { _lock.unlock(); } + +} + +void * +memory_region::realloc(void *mem, size_t size) { + if (_synchronized) { _lock.lock(); } + if (!mem) { + _live_allocations++; + } + void *newMem = _srv->realloc(mem, size); +#ifdef TRACK_ALLOCATIONS + if (_allocation_list.replace(mem, newMem) == false) { + printf("realloc: ptr 0x%" PRIxPTR " is not in allocation_list\n", + (uintptr_t) mem); + _srv->fatal("not in allocation_list", __FILE__, __LINE__, ""); + } +#endif + if (_synchronized) { _lock.unlock(); } + return newMem; +} + +void * +memory_region::malloc(size_t size) { + if (_synchronized) { _lock.lock(); } + _live_allocations++; + void *mem = _srv->malloc(size); +#ifdef TRACK_ALLOCATIONS + _allocation_list.append(mem); +#endif + if (_synchronized) { _lock.unlock(); } + return mem; +} + +void * +memory_region::calloc(size_t size) { + if (_synchronized) { _lock.lock(); } + _live_allocations++; + void *mem = _srv->malloc(size); + memset(mem, 0, size); +#ifdef TRACK_ALLOCATIONS + _allocation_list.append(mem); +#endif + if (_synchronized) { _lock.unlock(); } + return mem; +} + +memory_region::~memory_region() { + if (_live_allocations == 0) { + return; + } + char msg[128]; + snprintf(msg, sizeof(msg), + "leaked memory in rust main loop (%" PRIuPTR " objects)", + _live_allocations); +#ifdef TRACK_ALLOCATIONS + for (size_t i = 0; i < _allocation_list.size(); i++) { + if (_allocation_list[i] != NULL) { + printf("allocation 0x%" PRIxPTR " was not freed\n", + (uintptr_t) _allocation_list[i]); + } + } +#endif + _srv->fatal(msg, __FILE__, __LINE__, "%d objects", _live_allocations); +} diff --git a/src/rt/memory_region.h b/src/rt/memory_region.h new file mode 100644 index 00000000000..3411d867ebf --- /dev/null +++ b/src/rt/memory_region.h @@ -0,0 +1,37 @@ +/* + * The Rust runtime uses memory regions to provide a primitive level of + * memory management and isolation between tasks, and domains. + * + * TODO: Implement a custom lock-free malloc / free instead of relying solely + * on the standard malloc / free. + */ + +#ifndef MEMORY_REGION_H +#define MEMORY_REGION_H + +#include "sync/spin_lock.h" + +class rust_srv; + +class memory_region { +private: + rust_srv *_srv; + memory_region *_parent; + size_t _live_allocations; + array_list _allocation_list; + const bool _synchronized; + spin_lock _lock; +public: + enum memory_region_type { + LOCAL = 0x1, SYNCHRONIZED = 0x2 + }; + memory_region(rust_srv *srv, bool synchronized); + memory_region(memory_region *parent); + void *malloc(size_t size); + void *calloc(size_t size); + void *realloc(void *mem, size_t size); + void free(void *mem); + virtual ~memory_region(); +}; + +#endif /* MEMORY_REGION_H */ diff --git a/src/rt/rust.h b/src/rt/rust.h index 97f5c0d448d..3c534c9444b 100644 --- a/src/rt/rust.h +++ b/src/rt/rust.h @@ -21,11 +21,6 @@ #include "rust_srv.h" -inline void *operator new(size_t size, rust_srv *srv) -{ - return srv->malloc(size); -} - /* * Local Variables: * fill-column: 78; diff --git a/src/rt/rust_builtin.cpp b/src/rt/rust_builtin.cpp index 6b1f7d198c4..276ea5582d0 100644 --- a/src/rt/rust_builtin.cpp +++ b/src/rt/rust_builtin.cpp @@ -38,7 +38,7 @@ last_os_error(rust_task *task) { #endif size_t fill = strlen(buf) + 1; size_t alloc = next_power_of_two(sizeof(rust_str) + fill); - void *mem = dom->malloc(alloc); + void *mem = dom->malloc(alloc, memory_region::LOCAL); if (!mem) { task->fail(1); return NULL; @@ -134,7 +134,7 @@ str_alloc_with_data(rust_task *task, { rust_dom *dom = task->dom; size_t alloc = next_power_of_two(sizeof(rust_str) + n_bytes); - void *mem = dom->malloc(alloc); + void *mem = dom->malloc(alloc, memory_region::LOCAL); if (!mem) return NULL; rust_str *st = new (mem) rust_str(dom, alloc, fill, d); diff --git a/src/rt/rust_dom.cpp b/src/rt/rust_dom.cpp index c41e26881fc..67a1be3644d 100644 --- a/src/rt/rust_dom.cpp +++ b/src/rt/rust_dom.cpp @@ -13,6 +13,8 @@ rust_dom::rust_dom(rust_srv *srv, rust_crate const *root_crate, root_crate(root_crate), _log(srv, this), srv(srv), + local_region(&srv->local_region), + synchronized_region(&srv->synchronized_region), name(name), running_tasks(this), blocked_tasks(this), @@ -144,36 +146,65 @@ rust_dom::fail() { } void * -rust_dom::malloc(size_t sz) { - void *p = srv->malloc(sz); - I(this, p); - log(rust_log::MEM, - "%s @0x%" PRIxPTR " rust_dom::malloc(%d) -> 0x%" PRIxPTR, - name, (uintptr_t) this, sz, p); - return p; +rust_dom::malloc(size_t size) { + return malloc(size, memory_region::LOCAL); } void * -rust_dom::calloc(size_t sz) { - void *p = this->malloc(sz); - memset(p, 0, sz); - return p; +rust_dom::malloc(size_t size, memory_region::memory_region_type type) { + if (type == memory_region::LOCAL) { + return local_region.malloc(size); + } else if (type == memory_region::SYNCHRONIZED) { + return synchronized_region.malloc(size); + } + return NULL; } void * -rust_dom::realloc(void *p, size_t sz) { - void *p1 = srv->realloc(p, sz); - I(this, p1); - log(rust_log::MEM, "rust_dom::realloc(0x%" PRIxPTR ", %d) -> 0x%" PRIxPTR, - p, sz, p1); - return p1; +rust_dom::calloc(size_t size) { + return calloc(size, memory_region::LOCAL); +} + +void * +rust_dom::calloc(size_t size, memory_region::memory_region_type type) { + if (type == memory_region::LOCAL) { + return local_region.calloc(size); + } else if (type == memory_region::SYNCHRONIZED) { + return synchronized_region.calloc(size); + } + return NULL; +} + +void * +rust_dom::realloc(void *mem, size_t size) { + return realloc(mem, size, memory_region::LOCAL); +} + +void * +rust_dom::realloc(void *mem, size_t size, + memory_region::memory_region_type type) { + if (type == memory_region::LOCAL) { + return local_region.realloc(mem, size); + } else if (type == memory_region::SYNCHRONIZED) { + return synchronized_region.realloc(mem, size); + } + return NULL; } void -rust_dom::free(void *p) { - log(rust_log::MEM, "rust_dom::free(0x%" PRIxPTR ")", p); - I(this, p); - srv->free(p); +rust_dom::free(void *mem) { + free(mem, memory_region::LOCAL); +} + +void +rust_dom::free(void *mem, memory_region::memory_region_type type) { + log(rust_log::MEM, "rust_dom::free(0x%" PRIxPTR ")", mem); + if (type == memory_region::LOCAL) { + local_region.free(mem); + } else if (type == memory_region::SYNCHRONIZED) { + synchronized_region.free(mem); + } + return; } #ifdef __WIN32__ @@ -264,7 +295,6 @@ void rust_dom::send_message(rust_message *message) { message, &_incoming_message_queue, this); - A(this, message->dom == this, "Message owned by non-local domain."); _incoming_message_queue.enqueue(message); } @@ -277,7 +307,8 @@ void rust_dom::drain_incoming_message_queue() { log(rust_log::COMM, "<== processing incoming message \"%s\" 0x%" PRIxPTR, message->label, message); message->process(); - delete message; + message->~rust_message(); + this->synchronized_region.free(message); } } @@ -322,8 +353,7 @@ rust_dom::get_port_proxy_synchronized(rust_port *port) { * Returns NULL if no tasks can be scheduled. */ rust_task * -rust_dom::schedule_task() -{ +rust_dom::schedule_task() { I(this, this); // FIXME: in the face of failing tasks, this is not always right. // I(this, n_live_tasks() > 0); diff --git a/src/rt/rust_dom.h b/src/rt/rust_dom.h index 3aaf635a622..d2b6e57033b 100644 --- a/src/rt/rust_dom.h +++ b/src/rt/rust_dom.h @@ -25,6 +25,8 @@ struct rust_dom rust_crate const *root_crate; rust_log _log; rust_srv *srv; + memory_region local_region; + memory_region synchronized_region; const char *const name; ptr_vec running_tasks; ptr_vec blocked_tasks; @@ -58,10 +60,15 @@ struct rust_dom template void logptr(char const *msg, T* ptrval); void fail(); - void *malloc(size_t sz); - void *calloc(size_t sz); - void *realloc(void *data, size_t sz); - void free(void *p); + void *malloc(size_t size); + void *malloc(size_t size, memory_region::memory_region_type type); + void *calloc(size_t size); + void *calloc(size_t size, memory_region::memory_region_type type); + void *realloc(void *mem, size_t size); + void *realloc(void *mem, size_t size, + memory_region::memory_region_type type); + void free(void *mem); + void free(void *mem, memory_region::memory_region_type type); void send_message(rust_message *message); void drain_incoming_message_queue(); diff --git a/src/rt/rust_internal.h b/src/rt/rust_internal.h index 67787719444..d6618873dff 100644 --- a/src/rt/rust_internal.h +++ b/src/rt/rust_internal.h @@ -173,25 +173,7 @@ check_null(rust_dom *dom, T value, char const *expr, #define CHECK_NULL(dom, e) (check_null(dom, e, #e, __FILE__, __LINE__)) -inline void *operator new(size_t sz, void *mem) { - return mem; -} - -inline void *operator new(size_t sz, rust_dom *dom) { - return dom->malloc(sz); -} - -inline void *operator new[](size_t sz, rust_dom *dom) { - return dom->malloc(sz); -} - -inline void *operator new(size_t sz, rust_dom &dom) { - return dom.malloc(sz); -} - -inline void *operator new[](size_t sz, rust_dom &dom) { - return dom.malloc(sz); -} +#include "memory.h" struct rust_timer diff --git a/src/rt/rust_message.cpp b/src/rt/rust_message.cpp index 1de804c9dec..b6b7fbf07e0 100644 --- a/src/rt/rust_message.cpp +++ b/src/rt/rust_message.cpp @@ -3,7 +3,8 @@ rust_message:: rust_message(const char* label, rust_task *source, rust_task *target) : - dom(target->dom), label(label), + label(label), + _dom(target->dom), _source(source), _target(target) { } @@ -12,12 +13,12 @@ rust_message::~rust_message() { } void rust_message::process() { - I(dom, false); + I(_dom, false); } rust_proxy * rust_message::get_source_proxy() { - return dom->get_task_proxy(_source); + return _dom->get_task_proxy(_source); } notify_message:: @@ -50,8 +51,9 @@ send(notification_type type, const char* label, rust_task *source, rust_proxy *target) { rust_task *target_task = target->delegate(); rust_dom *target_domain = target_task->dom; - notify_message *message = new (target_domain) - notify_message(type, label, source, target_task); + notify_message *message = + new (target_domain, memory_region::SYNCHRONIZED) notify_message(type, + label, source, target_task); target_domain->send_message(message); } @@ -83,9 +85,10 @@ send(uint8_t *buffer, size_t buffer_sz, const char* label, rust_task *source, rust_task *target_task = target->delegate(); rust_port *target_port = port->delegate(); rust_dom *target_domain = target_task->dom; - data_message *message = new (target_domain) - data_message(buffer, buffer_sz, label, source, - target_task, target_port); + data_message *message = + new (target_domain, memory_region::SYNCHRONIZED) + data_message(buffer, buffer_sz, label, source, + target_task, target_port); target_domain->send_message(message); } diff --git a/src/rt/rust_message.h b/src/rt/rust_message.h index b7b8568a628..6d986acf5d2 100644 --- a/src/rt/rust_message.h +++ b/src/rt/rust_message.h @@ -9,12 +9,11 @@ /** * Abstract base class for all message types. */ -class rust_message : public lock_free_queue_node, - public dom_owned { +class rust_message : public lock_free_queue_node { public: - rust_dom *dom; const char* label; private: + rust_dom *_dom; rust_task *_source; protected: rust_task *_target; @@ -70,7 +69,7 @@ public: data_message(uint8_t *buffer, size_t buffer_sz, const char* label, rust_task *source, rust_task *target, rust_port *port); - ~data_message(); + virtual ~data_message(); void process(); /** diff --git a/src/rt/rust_srv.cpp b/src/rt/rust_srv.cpp index 9239f643ebe..f2dfef63ab6 100644 --- a/src/rt/rust_srv.cpp +++ b/src/rt/rust_srv.cpp @@ -5,72 +5,31 @@ #include "rust_internal.h" #include "rust_srv.h" -#define TRACK_ALLOCATIONS - -rust_srv::rust_srv() : _live_allocations(0) { +rust_srv::rust_srv() : + local_region(this, false), + synchronized_region(this, true) { // Nop. } rust_srv::~rust_srv() { - if (_live_allocations != 0) { - char msg[128]; - snprintf(msg, sizeof(msg), - "leaked memory in rust main loop (%" PRIuPTR " objects)", - _live_allocations); -#ifdef TRACK_ALLOCATIONS - for (size_t i = 0; i < _allocation_list.size(); i++) { - if (_allocation_list[i] != NULL) { - printf("allocation 0x%" PRIxPTR " was not freed\n", - (uintptr_t) _allocation_list[i]); - } - } -#endif - fatal(msg, __FILE__, __LINE__, ""); - } -} - -void * -rust_srv::malloc(size_t bytes) { - ++_live_allocations; - void * val = ::malloc(bytes); -#ifdef TRACK_ALLOCATIONS - _allocation_list.append(val); -#endif - return val; -} - -void * -rust_srv::realloc(void *p, size_t bytes) { - if (!p) { - _live_allocations++; - } - void * val = ::realloc(p, bytes); -#ifdef TRACK_ALLOCATIONS - if (_allocation_list.replace(p, val) == false) { - printf("realloc: ptr 0x%" PRIxPTR " is not in allocation_list\n", - (uintptr_t) p); - fatal("not in allocation_list", __FILE__, __LINE__, ""); - } -#endif - return val; + // Nop. } void rust_srv::free(void *p) { -#ifdef TRACK_ALLOCATIONS - if (_allocation_list.replace(p, NULL) == false) { - printf("free: ptr 0x%" PRIxPTR " is not in allocation_list\n", - (uintptr_t) p); - fatal("not in allocation_list", __FILE__, __LINE__, ""); - } -#endif - if (_live_allocations < 1) { - fatal("live_allocs < 1", __FILE__, __LINE__, ""); - } - _live_allocations--; ::free(p); } +void * +rust_srv::malloc(size_t bytes) { + return ::malloc(bytes); +} + +void * +rust_srv::realloc(void *p, size_t bytes) { + return ::realloc(p, bytes); +} + void rust_srv::log(char const *msg) { printf("rt: %s\n", msg); diff --git a/src/rt/rust_srv.h b/src/rt/rust_srv.h index b25e5e75f4e..e617c002576 100644 --- a/src/rt/rust_srv.h +++ b/src/rt/rust_srv.h @@ -5,11 +5,13 @@ #ifndef RUST_SRV_H #define RUST_SRV_H +#include "sync/spin_lock.h" +#include "memory_region.h" + class rust_srv { -private: - size_t _live_allocations; - array_list _allocation_list; public: + memory_region local_region; + memory_region synchronized_region; virtual void log(char const *msg); virtual void fatal(char const *expression, char const *file, @@ -21,12 +23,12 @@ public: size_t line, char const *format, ...); + virtual void free(void *); virtual void *malloc(size_t); virtual void *realloc(void *, size_t); - virtual void free(void *); - virtual rust_srv *clone(); rust_srv(); virtual ~rust_srv(); + virtual rust_srv *clone(); }; #endif /* RUST_SRV_H */