iris/invoke.ccp

#pypp 0
// Iris: micro-kernel for a capability-based operating system.
// invoke.ccp: Capability invocation and kernel responses.
// Copyright 2009 Bas Wijnen <wijnen@debian.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

#include "kernel.hh"

static void log_message (char const *prefix, unsigned target, unsigned pdata, kCapability::Context *c):
	if !dbg_code.l:
		dbg_log (prefix)
		dbg_log (": ")
		dbg_log_num ((unsigned)old_current)
		dbg_log (" > ")
		dbg_log_num (target)
		dbg_log (":")
		dbg_log_num (pdata)
		dbg_log ("/")
		dbg_log_num (c->data[0].l)
		dbg_log (" ")
		dbg_log_num (c->data[0].h)
		dbg_log (",")
		dbg_log_num (c->data[1].l)
		dbg_log (" ")
		dbg_log_num (c->data[1].h)
		dbg_log (",")
		dbg_log_num (pdata)
		dbg_log (";")
		if c->reply.valid ():
			dbg_log_num ((unsigned)c->reply->target)
			dbg_log (":")
			dbg_log_num (c->reply->protected_data.l)
		dbg_log (",")
		if c->arg.valid ():
			dbg_log_num ((unsigned)c->arg->target)
			dbg_log (":")
			dbg_log_num (c->arg->protected_data.l)
		dbg_log ("\n")

void kThread::raise (unsigned code, unsigned data):
	dpanic (code, "raise")
	dbg_log ("raise ")
	dbg_log_num ((unsigned)current)
	dbg_log_char ('/')
	if code < Kernel::NUM_EXCEPTION_CODES:
		dbg_log (Kernel::exception_name[code])
	else:
		dbg_log ("invalid code:")
		dbg_log_num (code)
	dbg_log_char ('/')
	dbg_log_num (data)
	dbg_log_char ('\n')
	unrun ()
	if slots < 1 || !caps[0] || !caps[0]->cap (0)->target:
		return
	kCapability::Context c
	c.data[0] = Kernel::Num (code, data)
	caps[0]->cap (0)->invoke (&c)

// From user-provided, thus untrusted, data, find a capability.
kCapRef kThread::find_capability (unsigned code, bool *copy):
	*copy = code & CAP_COPY
	unsigned c = code & ~CAP_COPY
	unsigned slot = c >> 16
	unsigned num = c & 0xffff
	if slot >= slots || !caps[slot] || num >= caps[slot]->size:
		if c != CAP_NONE:
			dpanic (code, "debug")
			dbg_log_num ((unsigned)old_current)
			dbg_log (": invalid capability ")
			dbg_log_num (code)
			dbg_log_char ('\n')
			dbg_log_num (num)
			dbg_log_char (':')
			dbg_log_num (slot)
			dbg_log (" > ")
			if caps[slot]:
				dbg_log_num (caps[slot]->size)
			else:
				dbg_log ("no caps")
			dbg_log_char ('\n')
		return kCapRef ()
	return kCapRef (caps[slot], num)

// Try to deliver a message.
bool kReceiver::try_deliver ():
	if !messages:
		return false
	if !owner || !owner->is_waiting ():
		return false
	kMessage *m = last_message
	if protected_only:
		for ; m; m = (kMessage *)m->prev:
			if m->protected_data.value () == reply_protected_data.value ():
				protected_only = false
				break
		if !m:
			return false
	bool dummy
	kCapRef c = owner->find_capability (owner->recv_reply, &dummy)
	if c.valid ():
		c.clone (kCapRef (&m->caps, 0), true)
	c = owner->find_capability (owner->recv_arg, &dummy)
	if c.valid ():
		c.clone (kCapRef (&m->caps, 1), true)
	kThread_arch_receive (owner, m->protected_data, m->data)
	address_space->free_message (this, m)
	owner->unwait ()
	return true

// Send a message to a receiver; try to deliver it immediately.
bool kReceiver::send_message (Kernel::Num protected_data, kCapability::Context *c):
	//log_message ("send_message", (unsigned)this, protected_data.l, c)
	if owner && owner->is_waiting () && (protected_data.value () == reply_protected_data.value () || !protected_only):
		if protected_only:
			protected_only = false
		bool dummy
		kCapRef cap = owner->find_capability (owner->recv_reply, &dummy)
		if cap.valid ():
			cap.clone (c->reply, c->copy[0])
		cap = owner->find_capability (owner->recv_arg, &dummy)
		if cap.valid ():
			cap.clone (c->arg, c->copy[1])
		kThread_arch_receive (owner, protected_data, c->data)
		owner->unwait ()
		return true
	// The owner was not waiting, or it was not possible to deliver the message.  Put it in the queue.
	kMessage *msg = NULL;
	if queue_limit:
		msg = address_space->alloc_message (this)
		if msg:
			--queue_limit
	if !msg:
		// TODO: use sender-provided storage.
	if !msg:
		return false
	msg->protected_data = protected_data
	if protected_only && protected_data.value () == reply_protected_data.value ():
		// Put this message in the end (where it will be first seen).  Clear the protected_only flag.
		protected_only = false
		if msg->next:
			((kMessage *)msg->next)->prev = NULL
			messages = (kMessage *)msg->next
			msg->next = NULL
			msg->prev = last_message
			((kMessage *)msg->prev)->next = msg
			last_message = msg
	for unsigned i = 0; i < 2; ++i:
		msg->data[i] = c->data[i]
	msg->caps.clone (0, c->reply, c->copy[0])
	msg->caps.clone (1, c->arg, c->copy[1])
	return true

static kCapability::Context *context
// reply_caps is the source of a receiver-generated reply capability.
// replied_caps is the source of kernel-generated capabilities which are used as arguments in a reply.
static kCaps reply_caps, replied_caps
static kReceiver *reply_target
static Kernel::Num reply_protected

static void reply_num (unsigned num1, unsigned num2 = 0, unsigned num3 = 0):
	kCapability::Context c
	c.data[0] = Kernel::Num (num1, num2)
	c.data[1] = num3
	if reply_target:
		reply_target->send_message (reply_protected, &c)
	else:
		dpanic (0, "nothing to reply to")

static void reply_cap (unsigned target, Kernel::Num protected_data, kCapRef *ref, unsigned num = 0):
	replied_caps.set (0, (kReceiver *)target, protected_data, kCapRef (), ref)
	kCapability::Context c
	c.arg = kCapRef (&replied_caps, 0)
	c.copy[1] = true
	c.data[0] = Kernel::Num (num, 0)
	if reply_target:
		reply_target->send_message (reply_protected, &c)
	else:
		dpanic (0, "nothing to reply to")
	c.arg->invalidate ()

static void receiver_invoke (unsigned cmd, unsigned target, Kernel::Num protected_data, kCapability::Context *c):
	kReceiver *receiver = (kReceiver *)protected_data.l
	switch cmd:
		case Kernel::Receiver::SET_OWNER & REQUEST_MASK:
			if !c->arg.valid ():
				reply_num (~0)
				return
			unsigned cap = (unsigned)c->arg->target
			if cap != (CAPTYPE_THREAD | CAP_MASTER) && cap != (CAPTYPE_THREAD | Kernel::Thread::SET_OWNER):
				// FIXME: This makes it impossible to use a fake kThread capability.
				return
			receiver->own ((kThread *)c->arg->protected_data.l)
			break
		case Kernel::Receiver::CREATE_CAPABILITY & REQUEST_MASK:
			reply_cap ((unsigned)receiver, c->data[1], &receiver->capabilities)
			return
		case Kernel::Receiver::CREATE_CALL_CAPABILITY & REQUEST_MASK:
			reply_cap (CAPTYPE_RECEIVER | (c->data[0].h ? Kernel::Receiver::CALL_ASYNC : Kernel::Receiver::CALL), protected_data, &((kObject *)protected_data.l)->refs)
			return
		case Kernel::Receiver::GET_REPLY_PROTECTED_DATA & REQUEST_MASK:
			reply_num (receiver->reply_protected_data.l, receiver->reply_protected_data.h, receiver->protected_only ? 1 : 0)
			return
		case Kernel::Receiver::SET_REPLY_PROTECTED_DATA & REQUEST_MASK:
			receiver->reply_protected_data = c->data[1]
			break
		case Kernel::Receiver::GET_ALARM & REQUEST_MASK:
			reply_num (receiver->alarm_count)
			return
		case Kernel::Receiver::SET_ALARM & REQUEST_MASK:
		case Kernel::Receiver::ADD_ALARM & REQUEST_MASK:
			unsigned old = receiver->alarm_count
			if cmd == Kernel::Receiver::SET_ALARM & REQUEST_MASK:
				receiver->alarm_count = c->data[1].l
			else:
				receiver->alarm_count += c->data[1].l
			if (old == ~0) ^ (receiver->alarm_count == ~0):
				// The alarm stopped or started.
				if old == ~0:
					// It started.
					receiver->prev_alarm = NULL
					receiver->next_alarm = first_alarm
					if receiver->next_alarm:
						receiver->next_alarm->prev_alarm = receiver
					first_alarm = receiver
				else:
					// It stopped.
					if receiver->prev_alarm:
						receiver->prev_alarm->next_alarm = receiver->next_alarm
					else:
						first_alarm = receiver->next_alarm
					if receiver->next_alarm:
						receiver->next_alarm->prev_alarm = receiver->prev_alarm
			reply_num (receiver->alarm_count)
			return
		default:
			dpanic (0, "invalid receiver operation")
			reply_num (Kernel::ERR_INVALID_OPERATION)
			return
	reply_num (0)

static void memory_invoke (unsigned cmd, unsigned target, Kernel::Num protected_data, kCapability::Context *c):
	kMemory *mem = (kMemory *)protected_data.l
	switch cmd:
		case Kernel::Memory::CREATE & REQUEST_MASK:
			switch c->data[0].h:
				case CAPTYPE_RECEIVER:
					kReceiver *ret = mem->alloc_receiver ()
					if ret:
						reply_cap (CAPTYPE_RECEIVER | CAP_MASTER, (unsigned)ret, &ret->refs)
					else:
						dpanic (0x03311992, "out of memory creating receiver")
						reply_num (Kernel::ERR_OUT_OF_MEMORY)
					return
				case CAPTYPE_MEMORY:
					kMemory *ret = mem->alloc_memory ()
					if ret:
						reply_cap (CAPTYPE_MEMORY | CAP_MASTER, (unsigned)ret, &ret->refs)
					else:
						dpanic (0x13311992, "out of memory creating memory")
						reply_num (Kernel::ERR_OUT_OF_MEMORY)
					return
				case CAPTYPE_THREAD:
					kThread *ret = mem->alloc_thread (c->data[1].l)
					if ret:
						reply_cap (CAPTYPE_THREAD | CAP_MASTER, (unsigned)ret, &ret->refs)
					else:
						dpanic (0x23311992, "out of memory creating thread")
						reply_num (Kernel::ERR_OUT_OF_MEMORY)
					return
				case CAPTYPE_PAGE:
					kPage *ret = mem->alloc_page ()
					if ret:
						reply_cap (CAPTYPE_PAGE | CAP_MASTER, (unsigned)ret, &ret->refs)
					else:
						dpanic (0x33311992, "out of memory creating page")
						reply_num (Kernel::ERR_OUT_OF_MEMORY)
					return
				case CAPTYPE_CAPS:
					kCaps *ret = mem->alloc_caps (c->data[1].l)
					if ret:
						reply_cap (CAPTYPE_CAPS | CAP_MASTER, (unsigned)ret, &ret->refs)
					else:
						dpanic (0x43311992, "out of memory creating caps")
						reply_num (Kernel::ERR_OUT_OF_MEMORY)
					return
				default:
					dpanic (0, "invalid create type")
					reply_num (~0)
					return
			break
		case Kernel::Memory::DESTROY & REQUEST_MASK:
			if !c->arg.valid () || (unsigned)c->arg->target & ~KERNEL_MASK || !c->arg->target || ((kObject *)c->arg->protected_data.l)->address_space != mem:
				reply_num (~0)
				return
			switch (unsigned)c->arg->target & CAPTYPE_MASK:
				case CAPTYPE_RECEIVER:
					mem->free_receiver ((kReceiver *)c->arg->protected_data.l)
					break
				case CAPTYPE_MEMORY:
					mem->free_memory ((kMemory *)c->arg->protected_data.l)
					break
				case CAPTYPE_THREAD:
					mem->free_thread ((kThread *)c->arg->protected_data.l)
					break
				case CAPTYPE_PAGE:
					mem->free_page ((kPage *)c->arg->protected_data.l)
					break
				case CAPTYPE_CAPS:
					mem->free_caps ((kCaps *)c->arg->protected_data.l)
					break
				default:
					panic (0x55228930, "invalid case")
					return
			break
		case Kernel::Memory::LIST & REQUEST_MASK:
			// TODO
			break
		case Kernel::Memory::MAP & REQUEST_MASK:
			// FIXME: this should work for fake pages as well.
			if !c->arg.valid () || (unsigned)c->arg->target & ~KERNEL_MASK || ((unsigned)c->arg->target & CAPTYPE_MASK) != CAPTYPE_PAGE:
				dpanic (0x22993341, "Trying to map non-page")
				reply_num (~0)
				return
			kPage *page = (kPage *)c->arg->protected_data.l
			if page->address_space != mem:
				dpanic (0x52993341, "Trying to map foreign page")
				reply_num (~0)
				return
			bool readonly = c->data[1].l & (unsigned)c->arg->target & Kernel::Page::READONLY
			mem->map (page, c->data[1].l & PAGE_MASK, readonly)
			break
		case Kernel::Memory::MAPPING & REQUEST_MASK:
			bool readonly
			kPage *page = mem->get_mapping (c->data[1].l, &readonly)
			unsigned t = CAPTYPE_PAGE | CAP_MASTER
			if readonly:
				t |= Kernel::Page::READONLY
			reply_cap (t, (unsigned)page, &page->refs)
			return
		case Kernel::Memory::GET_LIMIT & REQUEST_MASK:
			reply_num (mem->limit)
			return
		case Kernel::Memory::SET_LIMIT & REQUEST_MASK:
			mem->limit = c->data[1].l
			break
		default:
			dpanic (0, "invalid memory operation")
			reply_num (Kernel::ERR_INVALID_OPERATION)
			return
	reply_num (0)

static void thread_invoke (unsigned cmd, unsigned target, Kernel::Num protected_data, kCapability::Context *c):
	kThread *thread = (kThread *)protected_data.l
	switch cmd:
		case Kernel::Thread::GET_INFO & REQUEST_MASK:
			switch c->data[0].h:
				case Kernel::Thread::PC:
					reply_num (thread->pc)
					return
				case Kernel::Thread::SP:
					reply_num (thread->sp)
					return
				case Kernel::Thread::FLAGS:
					reply_num (thread->flags)
					return
				default:
					reply_num (*kThread_arch_info (thread, c->data[0].h))
					return
		case Kernel::Thread::SET_INFO & REQUEST_MASK:
			unsigned *value
			switch c->data[1].l:
				case Kernel::Thread::PC:
					value = &thread->pc
					break
				case Kernel::Thread::SP:
					value = &thread->sp
					break
				case Kernel::Thread::FLAGS:
					// It is not possible to set the PRIV flag (but it can be reset).
					if c->data[1].l & Kernel::Thread::PRIV:
						c->data[1].h &= ~Kernel::Thread::PRIV
					value = &thread->flags
					if c->data[1].h & ~Kernel::Thread::USER_FLAGS:
						unsigned v = (*value & ~c->data[1].h) | (c->data[1].l & c->data[1].h)
						if (v & Kernel::Thread::WAITING) != (*value & Kernel::Thread::WAITING):
							if v & Kernel::Thread::WAITING:
								thread->wait ()
							else
								thread->unwait ()
						if (v & Kernel::Thread::RUNNING) != (*value & Kernel::Thread::RUNNING):
							if v & Kernel::Thread::RUNNING:
								thread->run ()
							else
								thread->unrun ()
					break
				default:
					value = kThread_arch_info (thread, c->data[1].l)
					break
			if value:
				*value = (*value & ~c->data[1].h) | (c->data[1].l & c->data[1].h)
			break
		case Kernel::Thread::USE_SLOT & REQUEST_MASK:
			if c->data[1].l >= thread->slots || !c->arg.valid ():
				dbg_send (5, 3)
				dpanic (c->data[1].l, "no argument given for USE_SLOT")
				reply_num (~0)
				return
			// FIXME: This doesn't allow using a fake caps.
			if (unsigned)c->arg->target != (CAPTYPE_CAPS | CAP_MASTER) && (unsigned)c->arg->target != (CAPTYPE_CAPS | Kernel::Caps::USE):
				dpanic (0, "argument for USE_SLOT is not a caps")
				reply_num (~0)
				return
			unsigned slot = c->data[1].l
			kCaps *new_caps = (kCaps *)c->arg->protected_data.l
			if slot >= thread->slots:
				dpanic (0, "using invalid slot")
				return
			if thread->caps[slot]:
				// TODO: invalidate slot.
			thread->caps[slot] = new_caps
			if new_caps:
				// TODO: link it into a list.
			break
		case Kernel::Thread::GET_CAPS & REQUEST_MASK:
			unsigned slot = c->data[1].l
			if slot < thread->slots:
				reply_cap (CAPTYPE_CAPS | CAP_MASTER, (unsigned)thread->caps[slot], &thread->caps[slot]->refs, thread->slots)
			else:
				reply_num (thread->slots)
			return
		case Kernel::Thread::SCHEDULE & REQUEST_MASK:
			do_schedule = true
			return
		default:
			if !(thread->flags & Kernel::Thread::PRIV):
				dpanic (0, "invalid thread operation")
				reply_num (Kernel::ERR_INVALID_OPERATION)
				return
			switch cmd:
				case Kernel::Thread::PRIV_REGISTER_INTERRUPT & REQUEST_MASK:
					arch_register_interrupt (c->data[1].l, c->arg.valid () && (((unsigned)c->arg->target) & ~REQUEST_MASK) == CAPTYPE_RECEIVER ? (kReceiver *)c->arg->protected_data.l : NULL)
					break
				case Kernel::Thread::PRIV_GET_TOP_MEMORY & REQUEST_MASK:
					reply_cap (CAPTYPE_MEMORY | CAP_MASTER, (unsigned)&top_memory, &top_memory.refs)
					return
				case Kernel::Thread::PRIV_MAKE_PRIV & REQUEST_MASK:
					if !c->arg.valid () || ((unsigned)c->arg->target) & ~REQUEST_MASK != CAPTYPE_THREAD:
						reply_num (~0)
						return
					((kThread *)c->arg->protected_data.l)->flags |= Kernel::Thread::PRIV
					break
				case Kernel::Thread::PRIV_ALLOC_RANGE & REQUEST_MASK:
					if !c->arg.valid () || ((unsigned)c->arg->target) & ~REQUEST_MASK != CAPTYPE_MEMORY:
						panic (0x54365435, "non-memory argument to alloc_range")
						reply_num (~0)
						return
					kMemory *mem = (kMemory *)c->arg->protected_data.l
					if !mem->use (c->data[1].l):
						dpanic (0x34365435, "out of memory during alloc_range")
						reply_num (Kernel::ERR_OUT_OF_MEMORY)
						return
					unsigned data = phys_alloc (c->data[1].l)
					if !data:
						mem->unuse (c->data[1].l)
						dpanic (0x14365435, "out of memory during alloc_range")
						reply_num (Kernel::ERR_OUT_OF_MEMORY)
						return
					reply_num (data & ~0xc0000000)
					return
				case Kernel::Thread::PRIV_ALLOC_PHYSICAL & REQUEST_MASK:
					if !c->arg.valid ():
						panic (0x71342134, "no argument provided for alloc physical")
						reply_num (~0)
						return
					if ((unsigned)c->arg->target & ~REQUEST_MASK) != CAPTYPE_PAGE:
						panic (0x21342134, "no page provided for alloc physical")
						reply_num (~0)
						return
					kPage *page = (kPage *)c->arg->protected_data.l
					page->forget ()
					if !(c->data[1].l & 2):
						if page->flags & Kernel::Page::PAYING:
							page->flags &= ~Kernel::Page::PAYING
							page->address_space->unuse () 
					else:
						// This is for mapping allocated ranges.  They are already paid for.  Record that.
						if page->flags & Kernel::Page::PAYING:
							page->address_space->unuse () 
						else:
							page->flags |= Kernel::Page::PAYING
					page->frame = c->data[1].l & PAGE_MASK
					page->flags |= Kernel::Page::FRAME
					if !(c->data[1].l & 1):
						page->flags |= Kernel::Page::UNCACHED
					if !(c->data[1].l & 2):
						page->flags |= Kernel::Page::PHYSICAL
					kPage_arch_update_mapping (page)
					break
				case Kernel::Thread::PRIV_PHYSICAL_ADDRESS & REQUEST_MASK:
					if !c->arg.valid () || ((unsigned)c->arg->target) & ~REQUEST_MASK != CAPTYPE_PAGE:
						dpanic (0x99049380, "invalid page for physical address")
						reply_num (~0)
						return
					kPage *page = (kPage *)c->arg->protected_data.l
					reply_num (page->frame & ~0xc0000000)
					return
				case Kernel::Thread::PRIV_PANIC & REQUEST_MASK:
					panic (c->data[1].l, "panic requested by thread")
					reply_num (~0)
					return
				case Kernel::Thread::DBG_SEND & REQUEST_MASK:
					dbg_send (c->data[1].l, c->data[1].h)
					break
				default:
					dpanic (0, "invalid priv thread operation")
					reply_num (Kernel::ERR_INVALID_OPERATION)
					return
	reply_num (0)
	return

static bool page_check_payment (kPage *page):
	kPage *p
	for p = page->share_prev; p; p = p->share_prev:
		if p->flags & Kernel::Page::PAYING:
			return true
	for p = page->share_next; p; p = p->share_next:
		if p->flags & Kernel::Page::PAYING:
			return true
	// No kPage is paying for this frame anymore.
	raw_pfree (page->frame)
	kPage *next
	for p = page->share_prev, next = p->share_prev; p; p = next, next = p->share_prev:
		p->frame = NULL
		p->share_prev = NULL
		p->share_next = NULL
		p->flags &= ~(Kernel::Page::SHARED | Kernel::Page::FRAME)
		kPage_arch_update_mapping (p)
	for p = page, next = p->share_next; p; p = next, next = p->share_next:
		p->frame = NULL
		p->share_prev = NULL
		p->share_next = NULL
		p->flags &= ~(Kernel::Page::SHARED | Kernel::Page::FRAME)
		kPage_arch_update_mapping (p)
	return false

static void page_invoke (unsigned cmd, unsigned target, Kernel::Num protected_data, kCapability::Context *c):
	kPage *page = (kPage *)protected_data.l
	switch cmd & ~Kernel::Page::READONLY:
		case Kernel::Page::SHARE & REQUEST_MASK:
			if !c->arg.valid ():
				// Cannot share without a target page.
				reply_num (~0)
				return
			if ((unsigned)c->arg->target & ~REQUEST_MASK) != CAPTYPE_PAGE:
				// FIXME: This makes it impossible to use a fake kPage capability.
				reply_num (~0)
				return
			kPage *t = (kPage *)c->arg->protected_data.l
			t->forget ()
			if c->data[0].h & Kernel::Page::READONLY || cmd & Kernel::Page::READONLY:
				t->flags |= Kernel::Page::READONLY
			if !(page->flags & Kernel::Page::FRAME):
				break
			if c->data[0].h & Kernel::Page::COPY:
				if ~t->flags & Kernel::Page::PAYING:
					break
				if !(c->data[0].h & Kernel::Page::FORGET) || page->flags & Kernel::Page::SHARED:
					unsigned *d = (unsigned *)page->frame
					if t == page:
						kPage *other = page->share_next ? page->share_next : page->share_prev
						if !other:
							kPage_arch_update_mapping (t)
							break
						if page->share_next:
							page->share_next->share_prev = page->share_prev
						if page->share_prev:
							page->share_prev->share_next = page->share_next
						page->share_next = NULL
						page->share_prev = NULL
						page_check_payment (other)
					else:
						t->flags |= Kernel::Page::FRAME
					t->frame = raw_zalloc ()
					for unsigned i = 0; i <= (c->data[0].h & ~PAGE_MASK); i += 4:
						((unsigned *)t->frame)[i >> 2] = d[i >> 2]
				else:
					if t != page:
						t->frame = page->frame
						t->flags |= Kernel::Page::FRAME
						page->frame = NULL
						page->flags &= ~Kernel::Page::FRAME
						kPage_arch_update_mapping (page)
				kPage_arch_update_mapping (t)
			else:
				if t == page:
					break
				if c->data[0].h & Kernel::Page::FORGET:
					if ~page->flags & Kernel::Page::SHARED:
						if t->flags & Kernel::Page::PAYING:
							t->frame = page->frame
							t->flags |= Kernel::Page::FRAME
						page->frame = NULL
						page->flags &= ~Kernel::Page::FRAME
						kPage_arch_update_mapping (page)
					else:
						t->share_prev = page->share_prev
						t->share_next = page->share_next
						if t->share_prev:
							t->share_prev->share_next = t
						if t->share_next:
							t->share_next->share_prev = t
						page->share_prev = NULL
						page->share_next = NULL
						page->forget ()
						page_check_payment (t)
				else:
					t->share_prev = page->share_prev
					t->share_next = page
					page->share_prev = t
					if t->share_prev:
						t->share_prev->share_next = t
				kPage_arch_update_mapping (t)
			break
		case Kernel::Page::SET_FLAGS & REQUEST_MASK:
			if cmd & Kernel::Page::READONLY:
				reply_num (~0)
				return
			// Always refuse to set reserved flags.
			c->data[1].h &= ~(Kernel::Page::PHYSICAL | Kernel::Page::UNCACHED)
			// Remember the old flags.
			unsigned old = page->flags
			// Compute the new flags.
			unsigned new_flags = (page->flags & ~c->data[1].h) | (c->data[1].l & c->data[1].h)

			// If we stop paying, see if the frame is still paid for.  If not, free it.
			if ~new_flags & old & Kernel::Page::PAYING:
				// Decrease the use counter in any case.
				page->address_space->unuse ()
				if !page_check_payment (page):
					new_flags &= ~Kernel::Page::FRAME

			// If we start paying, increase the use counter.
			if new_flags & ~old & Kernel::Page::PAYING:
				if !page->address_space->use():
					// If it doesn't work, refuse to set the flag, and refuse to allocate a frame.
					new_flags &= ~(Kernel::Page::PAYING | Kernel::Page::FRAME)
					if old & Kernel::Page::FRAME:
						new_flags |= Kernel::Page::FRAME

			// If we want a frame, see if we can get it.
			if ~old & new_flags & Kernel::Page::FRAME:
				kPage *p
				for p = page; p; p = p->share_prev:
					if p->flags & Kernel::Page::PAYING:
						break
				if !p:
					for p = page->share_next; p; p = p->share_next:
						if p->flags & Kernel::Page::PAYING:
							break
					if !p:
						new_flags &= ~Kernel::Page::FRAME
				// If we can get the new frame, get it.
				if ~old & new_flags & Kernel::Page::FRAME:
					page->frame = page->address_space->zalloc ()
					kPage_arch_update_mapping (page)
			break
		default:
			dpanic (0, "invalid page operation")
			reply_num (Kernel::ERR_INVALID_OPERATION)
			return
	reply_num (0)

static void print_cap (kCapRef cap, kCapRef self):
	if cap.deref () == self.deref ():
		dbg_log_char ('{')
	else:
		dbg_log_char ('[')
	dbg_log_num ((unsigned)cap.caps)
	dbg_log_char (':')
	dbg_log_num (cap.index, 1)
	if !cap.valid ():
		dbg_log_char ('!')
	else:
		dbg_log_char ('=')
		dbg_log_num (cap->protected_data.l)
	for kCapRef c = cap->children; c.valid (); c = c->sibling_next:
		print_cap (c, self)
	if cap.deref () == self.deref ():
		dbg_log_char ('}')
	else:
		dbg_log_char (']')

static void caps_invoke (unsigned cmd, unsigned target, Kernel::Num protected_data, kCapability::Context *c):
	kCaps *caps = (kCapsP)protected_data.l
	switch cmd:
		case Kernel::Caps::SET & REQUEST_MASK:
			if c->data[1].l >= caps->size:
				dpanic (0, "invalid index for set caps")
				return
			caps->clone (c->data[1].l, c->arg, c->copy[1])
			reply_num (0)
			return
		case Kernel::Caps::PRINT & REQUEST_MASK:
			if c->data[1].l >= caps->size:
				dpanic (0, "invalid caps for print")
				return
			kCapRef cap (caps, c->data[1].l)
			kCapRef orig (caps, c->data[1].l)
			while cap->parent.valid ():
				while cap->sibling_prev.valid ():
					if cap->parent.deref () != cap->sibling_prev->parent.deref ():
						dpanic (0, "parent problem in cap data")
						return
					if cap.deref () != cap->sibling_prev->sibling_next.deref ():
						dpanic (0, "prev error in cap data")
						return
					cap = cap->sibling_prev
				if cap->parent->children.deref () != cap.deref ():
					dpanic (0, "parent error in cap data")
					return
				cap = cap->parent
			while cap->sibling_prev.valid ():
				if cap->parent.deref () != cap->sibling_prev->parent.deref ():
					dpanic (0, "parent parent problem in cap data")
					return
				if cap.deref () != cap->sibling_prev->sibling_next.deref ():
					dpanic (0, "parent prev error in cap data")
					return
				cap = cap->sibling_prev
			while cap.valid ():
				print_cap (cap, orig)
				cap = cap->sibling_next
			dbg_log_char ('\n')
			return
		default:
			dpanic (0, "invalid caps operation")
			reply_num (Kernel::ERR_INVALID_OPERATION)
			return

static void kill_reply (kReceiver *r):
	kCapRef cap = r->refs
	while cap.valid ():
		kCapability *c = cap.deref ()
		cap = c->sibling_next
		if (unsigned)c->target == (CAPTYPE_RECEIVER | Kernel::Receiver::REPLY):
			c->invalidate ()

static void kernel_invoke (unsigned target, Kernel::Num protected_data, kCapability::Context *c):
	// Kernel calling convention:
	// data[0].l is the request.
	// caps[0] is the reply capability
	// other parameters' meanings depend on the operation.
	if target == (CAPTYPE_RECEIVER | Kernel::Receiver::CALL) || target == (CAPTYPE_RECEIVER | Kernel::Receiver::CALL_ASYNC):
		// This is a call capability.  reply is the capability to call.  It should be replaced by the reply capability.
		((kReceiver *)protected_data.l)->protected_only = target == (CAPTYPE_RECEIVER | Kernel::Receiver::CALL)
		if must_wait:
			old_current->wait ()
		if !reply_target:
			dpanic (0x54635675, "no target to call")
			return
		if ((unsigned)reply_target & ~KERNEL_MASK) != 0:
			// This is a user-implemented object.  Create a real reply capability.
			kReceiver *call_target = reply_target
			c->reply = kCapRef (&reply_caps, 0)
			c->reply.set ((kReceiver *)(CAPTYPE_RECEIVER | Kernel::Receiver::REPLY), protected_data, kCapRef (), &((kReceiver *)protected_data.l)->refs)
			c->copy[0] = true
			call_target->send_message (reply_protected, c)
			c->reply->invalidate ()
		else if (unsigned)reply_target == (CAPTYPE_RECEIVER | Kernel::Receiver::REPLY):
			// Reply capability: destroy all before invoke.
			kReceiver *r = (kReceiver *)reply_protected.l
			kill_reply (r)
			r->send_message (r->reply_protected_data, c)
		else:
			// Kernel call: don't create actual capablities.
			kCapRef call_target = c->reply
			c->reply.reset ()
			reply_target = (kReceiver *)protected_data.l
			reply_protected = reply_target->reply_protected_data
			kernel_invoke ((unsigned)call_target->target, call_target->protected_data, c)
		return
	if must_wait:
		old_current->wait ()
	if target == (CAPTYPE_RECEIVER | Kernel::Receiver::REPLY):
		// This is a reply capability.
		kReceiver *r = (kReceiver *)protected_data.l
		kill_reply (r)
		r->send_message (r->reply_protected_data, c)
		return
	if !target:
		return
	unsigned cmd
	if (target & REQUEST_MASK) == CAP_MASTER:
		if c->data[0].l & CAP_MASTER_CREATE:
			reply_cap (target | (c->data[0].l & REQUEST_MASK), protected_data, &((kObject *)protected_data.l)->refs)
			return
		cmd = c->data[0].l
		c->data[0].l = 0
	else:
		cmd = target
	cmd &= REQUEST_MASK
	switch target & CAPTYPE_MASK:
		case CAPTYPE_RECEIVER:
			receiver_invoke (cmd, target, protected_data, c)
			break
		case CAPTYPE_MEMORY:
			memory_invoke (cmd, target, protected_data, c)
			break
		case CAPTYPE_THREAD:
			thread_invoke (cmd, target, protected_data, c)
			break
		case CAPTYPE_PAGE:
			page_invoke (cmd, target, protected_data, c)
			break
		case CAPTYPE_CAPS:
			caps_invoke (cmd, target, protected_data, c)
			break
		default:
			panic (0x99337744, "invalid capability type invoked")
			return
	return

void invoke (kReceiverP target, Kernel::Num protected_data, kCapability::Context *c):
	//log_message ("invoke", (unsigned)target, protected_data.l, c)
	if (unsigned)target & ~KERNEL_MASK:
		// This is not a kernel capability: send a message to the receiver.
		if must_wait:
			old_current->wait ()
		target->send_message (protected_data, c)
		return
	// This is a kernel capability.  Use a function to allow optimized call capabilities.
	context = c
	if c->reply.valid ():
		reply_target = c->reply->target
		reply_protected = c->reply->protected_data
	else:
		reply_target = NULL
	kernel_invoke ((unsigned)target, protected_data, c)