#include "assert.h"
#include "tkm.h"
#include "unistd.h"
#include "memory.h"
#include "stdlib.h"
#include "stdio.h"
#include "strings.h"
#include "getopt.h"
#include "ulocks.h"
#include "mutex.h"
#include "time.h"
#include "signal.h"
#include "sys/types.h"
#include "sys/prctl.h"
#include "mbox.h"

static void sig(int);
static void node(int, int);

static int srpc(struct mbox *, tks_ch_t, void *, int, void *, int);
#define MAXTOKENS	10
#define MAXSERVICES	10
#define SERVICE1_ID	0
#define SERVICE2_ID	1
/* are we the service1 server? */
#define S1S()		(services[SERVICE1_ID].server_node)
#define AM_S1S(n)	(n == services[SERVICE1_ID].server_node)
#define S2S()		(services[SERVICE2_ID].server_node)
#define AM_S2S(n)	(n == services[SERVICE2_ID].server_node)

/*
 * protocol
 */
struct scheme {
	tks_ch_t handle;		/* client handle */
	struct mbox *return_addr;	/* return address for RPC */
};
/*
 * htonode - convert a handle to a node number
 */
#define htonode(h)	(h)
#define nodetoh(n)	(n)

#define TEST2_OBTAIN 1		/* client -> server RPC */
#define TEST2_REVOKE 2		/* server -> client 1-way */
#define TEST2_RETURN 3		/* client -> server 1-way */
#define TEST2_RECALL 4		/* server -> client 1-way */
#define TEST2_RETURN_RECALL 5	/* client -> server 1-way */
#define TEST2_GEN 6		/* client -> server RPC */
struct test2_req {
	struct scheme addr;		/* MUST be first!! */
	int op;
	int service_id;
	union {
		/* obtain request message */
		struct obtain_req {
			tk_set_t obtain;
			tk_set_t toreturn;
			int data[MAXTOKENS];
		} obtain_req;
		struct revoke_req {
			tk_set_t torevoke;
		} revoke_req;
		struct recall_req {
			tk_set_t torecall;
			int flag;
		} recall_req;
		struct return_req {
			tk_set_t toreturn;
			int data[MAXTOKENS];
			tk_set_t refuse;/* for TEST2_RETURN_RECALL only */
			tk_set_t eh;/* for TEST2_RETURN_RECALL only */
		} return_req;
		struct gen_req {
			int gen;
		} gen_req;
	} req_un;
};
struct test2_res {
	struct scheme addr;		/* MUST be first!! */
	int op;
	union {
		/* obtain response message */
		struct obtain_res {
			tk_set_t granted;
			tk_set_t already;
			int data[MAXTOKENS];
		} obtain_res;
		struct gen_res {
			tk_set_t existance;
		} gen_res;
	} res_un;
};

/*
 * XXX currently all services are really pretty much the same - they
 * share the same basic algorithms..
 */
struct service_info {
	int server_node;	/* node that will act as server */
	int service_id;
	int cmpl;		/* client multi-programming level */
	int nclasses;		/* # classes in token
				 * we use nclasses+1 as the existance token
				 */
	void (*wait)(struct test2_req *, int);
	void (*start)(void *);
	void **nsdata;		/* private data */
	tk_set_t alltk;		/* set of all tokens */
} services[MAXSERVICES];

#define S1_P(n) (services[SERVICE1_ID].nsdata[n])/* get private data for node */
#define S1_NCLASS()  (services[SERVICE1_ID].nclasses) /* get nclasses */
#define S1_ALLTK()  (services[SERVICE1_ID].alltk)

int nloops = 10;
pid_t ppid;
unsigned long ndoneclients;
int verbose;
/*
 * the following arena is used for all locks and semaphores for all nodes
 * and all services ...
 */
usptr_t *usptr;
char *Cmd;
int nodes;
int dumplog = 0;

static void service1_wait(struct test2_req *, int);
static void service1_start(void *);

int
main(int argc, char **argv)
{
	int c, i;
	int cmpl, nservices;
	int nclasses;
	unsigned long totclients;
	int nwaiters;

	setlinebuf(stdout);
	setlinebuf(stderr);
	Cmd = strdup(argv[0]);
	ppid = getpid();

	prctl(PR_COREPID, 0, 1);

	cmpl = 1;
	nodes = 0;
	nclasses = 1;

	while ((c = getopt(argc, argv, "TN:vLm:n:t:")) != EOF) 
		switch (c) {
		case 'T':
			__tk_tracenow = 1;
			break;
		case 'v':
			verbose = 1;
			break;
		case 't':
			nclasses = atoi(optarg);
			break;
		case 'L':
			dumplog++;
			break;
		case 'm':
			cmpl = atoi(optarg);
			break;
		case 'N':
			nodes = atoi(optarg);
			break;
		case 'n':
			nloops = atoi(optarg);
			break;
		default:
			fprintf(stderr, "illegal option %c\n", c);
			exit(1);
		}

	/*
	 * setup services
	 */
	for (i = 0; i < MAXSERVICES; i++)
		services[i].service_id = -1;
	nservices = 1;
	services[0].server_node = 0;
	services[0].service_id = SERVICE1_ID;
	services[0].cmpl = cmpl;
	services[0].nclasses = nclasses;
	services[0].wait = service1_wait;
	services[0].start = service1_start;
	services[0].nsdata = (void **)malloc(nodes * sizeof(void *));
	services[0].alltk = TK_NULLSET;
	/* alltk includes existance token */
	for (i = 0; i < S1_NCLASS()+1; i++)
		TK_COMBINE_SET(services[0].alltk, TK_MAKE(i, TK_READ|TK_WRITE|TK_SWRITE));

#ifdef LATER
	nservices++;
	services[1].server_node = 1;
	services[1].service_id = SERVICE2_ID;
	services[1].cmpl = cmpl;
	services[1].nclasses = nclasses;
#endif
	nodes = nodes < nservices ? nservices : nodes;

	/*
	 * XXX we really need more why??
	 * 1) each service has 'cmpl' client threads per node
	 * 2) Each service server needs a thread per potential client thread 
	 *	that is not on its node
	 * 3) each client not on the server needs a 'wait' thread.
	 * 4) Each server needs a thread to handle the GEN message from each
	 *	node.
	 */
	totclients = nodes * nservices * cmpl;
	nwaiters = nservices * ((nodes - 1) * cmpl);
	nwaiters *= 2;
	nwaiters += (nodes - 1);
	usconfig(CONF_INITUSERS, nwaiters + totclients + 1);

	sigset(SIGTERM, sig);
	sigset(SIGABRT, sig);
	sigset(SIGUSR1, sig);

	prctl(PR_SETEXITSIG, SIGTERM);
	/* initialize RPC */
	initmbox();

	/* call tkc_init to get arenas started up in correct sequence */
	tkc_init();
	tks_init();

	/*
	 * alloc an area for locks - note that all nodes 'share' this
	 * but that should be OK
	 */
	usconfig(CONF_ARENATYPE, US_SHAREDONLY);
	if ((usptr = usinit("/usr/tmp/tktest2")) == NULL) 
		abort();

	/* create 'nodes' */
	for (i = 0; i < nodes; i++)
		node(i, nodes);

	while (ndoneclients != totclients)
		sginap(100);

	if (dumplog)
		tk_printlog(stdout, 1000, dumplog > 2 ? TK_LOG_ALL :
			(dumplog > 1 ? TK_LOG_TS : 0));

	return 0;
}

static void
sig(int s)
{
	if (s == SIGABRT) {
		tk_printlog(stdout, 1000, dumplog > 2 ? TK_LOG_ALL :
			(dumplog > 1 ? TK_LOG_TS : 0));
		sigset(SIGABRT, SIG_DFL);
		abort();
	} else if (s == SIGUSR1) {
		tk_printlog(stdout, -1, dumplog > 2 ? TK_LOG_ALL :
			(dumplog > 1 ? TK_LOG_TS : 0));
		return;
	}
	exit(0);
}

/*
 * Node
 */
/*
 * do all waits for ootb messages - both client and server
 */
static void
node_wait(void *a)
{
	int node = (int)(ptrdiff_t)a;
	struct mbox *mbox;
	auto struct test2_req *req;
	auto int dyn;

	mbox = ntombox(node);
	for (;;) {
		readmbox(mbox, (void *)&req, &dyn);
		(services[req->service_id].wait)(req, node);
		if (dyn)
			freemboxmsg(req);
	}
}

static void
node(int mynode, int nodes)
{
	int i, j;
	struct mbox *mb;
	pid_t spid;

	mb = allocmbox();
	setmbox(mynode, mb);

	for (i = 0; i < MAXSERVICES; i++) {
		if (services[i].service_id < 0)
			continue;

		/* start up wait threads -
		 * if we are the server then we need 'cmpl' server threads
		 * 	per client node but no client wait threads.
		 * if we are a client we need 'cmpl' client wait threads
		 *	but no server wait threads ..
		 */
		if (services[i].server_node == mynode) {
			int ns;
			ns = (nodes - 1) * services[SERVICE1_ID].cmpl;
			/* one per node for GEN message */
			ns += (nodes - 1);
			for (j = 0; j < ns; j++) {
				if ((spid = sproc(node_wait, PR_SALL,
					    (void *)(ptrdiff_t)mynode)) < 0) {
					perror("sproc");
					exit(1);
				}
				if (verbose)
					printf("%s:started up server wait thread on node %d pid %d\n",
						Cmd, mynode, spid);
			}
		} else {
			int ns;
			ns = services[SERVICE1_ID].cmpl;
			for (j = 0; j < ns; j++) {
				if ((spid = sproc(node_wait, PR_SALL,
					   (void *)(ptrdiff_t)mynode)) < 0) {
					perror("sproc");
					exit(1);
				}
				if (verbose)
					printf("%s:started up client wait thread on node %d pid %d\n",
						Cmd, mynode, spid);
			}
		}

		if ((spid = sproc(services[i].start, PR_SALL, (void *)(ptrdiff_t)mynode)) < 0) {
			perror("sproc");
			exit(1);
		}
		if (verbose)
			printf("%s:started up service %d on node %d pid %d\n",
					Cmd, i, mynode, spid);
	}
}

/* remote data for clients */
struct cdata {
	TKC_DECL(cs, MAXTOKENS);
	int target[MAXTOKENS];
	ulock_t lock;		/* protect mp client threads */
	tks_ch_t h;		/* handle (contains node we're on) */
	int initted;		/* are tokens created? */
	int node;		/* node we're running on */
	usema_t *sync;		/* sync up threads */
	usema_t *sync2;		/* sync up threads */
	int ref;		/* ref count */
};

/* local data for server */
struct sdata {
	TKS_DECL(ss, MAXTOKENS);
	ulock_t lock;		/* protect remote and local threads */
	struct cdata *cp;	/* server's client data */
	usema_t *sync;		/* for recall operation */
	int gen;		/* which generation we're accepting requests
				 * for
				 */
	usema_t *gensync;	/* wait for generation */
	int nwaitgen;
	int ncl;		/* track # clients that have initialized */
};

#define INOT	1		/* not initialized */
#define IIS	2		/* in progress of initializing */
#define IAM	3		/* initialized */

int shadowtarget[MAXTOKENS];	/* to track 'target' */
char *levels[] = {
	"UNK0",
	"READ",
	"WRITE",
	"UNK3",
	"SWRITE",
	"UNK5",
	"UNK6",
	"UNK7"
};
static void client_obtain(void *, tk_set_t, tk_set_t, tk_set_t *, tk_set_t *);
static void client_revoke(void *, tk_set_t);
static void client_recall(void *, tk_set_t, tk_set_t, tk_set_t);
static void server_revoke(void *o, tks_ch_t h, tk_set_t r);
static void server_recall(void *o, tks_ch_t h, tk_set_t r, int);
static void service1_recalled(void *o, tk_set_t r, tk_set_t);
static void server_return(struct sdata *, struct test2_req *);

static void
service1_play(struct cdata *cp)
{
	int i, lev, class;
	struct timespec ts;
	int loops = 10;

	for (i = 0; i < loops; i++) {
		if (verbose)
			printf("client %d node %d loop %d\n",
					get_pid(), cp->node, i);
		/* get which token */
		lev = 1 << (rand() % 3);
		class = rand() % S1_NCLASS();
		tkc_acquire(cp->cs, TK_MAKE(class, lev));
		ussetlock(cp->lock);
		if (lev == TK_WRITE) {
			/*
			 * note that 2 threads in a client 'node'
			 * can both have the TK_WRITE level - the token
			 * module makes no claims within a node
			 */
			cp->target[class] = rand();
			shadowtarget[class] = cp->target[class];
		} else if (cp->target[class] != shadowtarget[class]) {
			fprintf(stderr, "ERROR:client:%d:0x%x class:%d cd.target %d shadowtarget %d\n",
				get_pid(),
				cp->h,
				class,
				cp->target[class], shadowtarget[class]);
			abort();
		}
		usunsetlock(cp->lock);
		ts.tv_sec = 0;
		ts.tv_nsec = (rand() % 4) * 1000 * 1000;
		nanosleep(&ts, NULL);
		tkc_release(cp->cs, TK_MAKE(class, lev));
	}
}

/*
 * service1_test - the real test code
 * This runs on each node and is multi-threaded
 */
static void
service1_test(int node)
{
	int i, j;
	tkc_ifstate_t iface;
	tks_ifstate_t svriface;
	struct sdata *sp;
	struct cdata *cp;
	tk_set_t existance;
	struct test2_req req;
	struct test2_res res;
	char cname[TK_NAME_SZ], sname[TK_NAME_SZ];

	iface.tkc_obtain = client_obtain;
	iface.tkc_revoke = client_revoke;
	iface.tkc_recall = client_recall;
	svriface.tks_revoke = server_revoke;
	svriface.tks_recall = server_recall;
	svriface.tks_recalled = service1_recalled;

	existance = TK_MAKE(S1_NCLASS(), TK_READ);

	for (i = 0; i < nloops; i++) {
		/*
		 * for server create both server and client token sets
		 */
		sprintf(cname, "Ctest2%d", i);
		sprintf(sname, "Stest2%d", i);
		if (AM_S1S(node)) {
			sp = S1_P(node);
			cp = sp->cp;
			ussetlock(cp->lock);
			if (cp->initted == INOT) {
				cp->initted = IIS;
				usunsetlock(cp->lock);

				/* create token set */
				tks_create(sname, sp->ss, (void *)sp, &svriface,
					S1_NCLASS() + 1);
				tkc_create(cname, cp->cs, (void *)cp, &iface,
					S1_NCLASS() + 1,
					TK_NULLSET);
				cp->initted = IAM;
				/*
				 * immediately give ourselves an existance token
				 */
				tkc_acquire(cp->cs, existance);
				/*
				 * we are now ready to handle generation 'i'
				 */
				ussetlock(cp->lock);
				sp->gen = i;
				sp->ncl = 0;
				for (j = 0; j < sp->nwaitgen; j++)
					usvsema(sp->gensync);
				sp->nwaitgen = 0;
				usunsetlock(cp->lock);
				/*
				 * now wait for all clients to 'register'
				 */
				while (sp->ncl < (nodes - 1)) {
					sginap(10);
				}
				
				/* wake up other server/client threads on our
				 * node
				 */
				for (j = 1; j < services[SERVICE1_ID].cmpl; j++)
					usvsema(cp->sync2);
			} else {
				usunsetlock(cp->lock);
				uspsema(cp->sync2);
			}
		} else {
			cp = S1_P(node);
			ussetlock(cp->lock);
			if (cp->initted == INOT) {
				cp->initted = IAM;
				usunsetlock(cp->lock);

				/*
				 * send message to server requesting permission
				 * to play in generation 'gen'
				 */
				req.req_un.gen_req.gen = i;
				req.op = TEST2_GEN;
				req.service_id = SERVICE1_ID;
				srpc(ntombox(S1S()), cp->h, &req, sizeof(req),
							&res, sizeof(res));
				/*
				 * since we now have existance token
				 * the previous generation must be done so
				 * we can initialize a new one
				 */
				tkc_create(cname, cp->cs, (void *)cp, &iface,
					S1_NCLASS() + 1,
					res.res_un.gen_res.existance);
				/* wake all other threads (on our node) up */
				for (j = 1; j < services[SERVICE1_ID].cmpl; j++)
					usvsema(cp->sync2);
			} else {
				usunsetlock(cp->lock);
				uspsema(cp->sync2);
			}
		}
		/*
		 * At this point all clients have an existance token
		 * so that no matter who finishes first, the server won't
		 * be able to complete the recall until everyone is
		 * at least out of 'play'
		 */
		service1_play(cp);

		/*
		 * all threads but the last one will wait
		 */
		ussetlock(cp->lock);
		assert(cp->ref > 0);
		if (--cp->ref != 0) {
			usunsetlock(cp->lock);
			uspsema(cp->sync);
			continue;
		}
		usunsetlock(cp->lock);

		/* now that we are done - release our 'existance' token */ 
		tkc_release(cp->cs, existance);
		
		/*
		 * we are last thread in our group
		 * The server thread starts a recall
		 */
		if (AM_S1S(node)) {
			/* recall and destroy token */
			tks_recall(sp->ss, existance);
			/* service1_recalled will be called when all done */
			uspsema(sp->sync);
			/* test that everything got sent back */
			for (j = 0; j < S1_NCLASS(); j++) {
				if (cp->target[j] != shadowtarget[j]) {
					fprintf(stderr, "ERROR:server:%d:class:%d cd.target %d shadowtarget %d\n",
						get_pid(),
						j,
						cp->target[j], shadowtarget[j]);
					abort();
				}
			}
			tks_destroy(sp->ss);
		}

		/* reset the ref count and go again */
		cp->ref = services[SERVICE1_ID].cmpl;
		cp->initted = INOT;
		/* wake all other threads (on our node) up */
		for (j = 1; j < services[SERVICE1_ID].cmpl; j++)
			usvsema(cp->sync);
	}
	test_then_add(&ndoneclients, 1);
	pause();

}

static void
service1_launch(void *a)
{
	service1_test((int)(ptrdiff_t)a);
	/* NOTREACHED */
}

void
service1_start(void *a)
{
	int mynode = (int)(ptrdiff_t)a;
	int i;
	struct sdata sd;
	struct cdata cd;	/* this is what we're manipulating */

	/* init a client data object */
	cd.h = nodetoh(mynode);	/* handle is just node number */
	cd.initted = INOT;
	cd.lock = usnewlock(usptr);
	cd.sync = usnewsema(usptr, 0);
	cd.sync2 = usnewsema(usptr, 0);
	cd.node = mynode;
	cd.ref = services[SERVICE1_ID].cmpl;

	if (AM_S1S(mynode)) {
		/* init server side */
		sd.lock = usnewlock(usptr);
		sd.sync = usnewsema(usptr, 0);
		sd.cp = &cd;
		/*
		 * to provide syncronization between loops, clients
		 * send a request to the server requesting to 'play' in a
		 * particular generation - that request will sleep
		 * until the server is ready
		 */
		sd.gensync = usnewsema(usptr, 0);
		sd.gen = 0;	/* "now serving" */
		sd.nwaitgen = 0;
		/* place local data in 'fake' private memory */
		S1_P(mynode) = &sd;
	} else {
		/* place local data in 'fake' private memory */
		S1_P(mynode) = &cd;
	}

	/*
	 * each client / server is multi-threaded
	 */
	for (i = 1; i < services[SERVICE1_ID].cmpl; i++) {
		if (sproc(service1_launch, PR_SALL,
				(void *)(ptrdiff_t)mynode) < 0) {
			perror("sproc");
			exit(1);
		}
	}
	service1_test(mynode);
	/* NOTREACHED */
}

/* ARGSUSED */
static void
client_obtain(void *o,
	tk_set_t obtain,
	tk_set_t toreturn,
	tk_set_t *already,
	tk_set_t *refused)
{
	struct test2_req req;
	struct test2_res res;
	struct cdata *cp = (struct cdata *)o;
	int i;

	if (AM_S1S(cp->node)) {
		/* we are server */
		struct sdata *sp = S1_P(cp->node);
		auto tk_set_t granted;

		if (toreturn != TK_NULLSET)
			tks_return(sp->ss, cp->h, toreturn);
		tks_obtain(sp->ss, cp->h, obtain, &granted, already);
		assert(obtain == granted);
		return;
	}

	/* remote client */
	/* send out obtain request message */
	req.req_un.obtain_req.obtain = obtain;
	req.req_un.obtain_req.toreturn = toreturn;
	req.op = TEST2_OBTAIN;
	req.service_id = SERVICE1_ID;
	for (i = 0; i < S1_NCLASS(); i++) {
		if (TK_IS_IN_SET(toreturn, TK_MAKE(i, TK_WRITE))) {
			/* returning write token - return data also */
			req.req_un.obtain_req.data[i] = cp->target[i];
		}
	}

	srpc(ntombox(S1S()), cp->h, &req, sizeof(req), &res, sizeof(res));

	if (req.op != res.op) {
		fprintf(stderr, "client_obtain: srpc return message op wrong!\n");
		exit(1);
	}

	for (i = 0; i < S1_NCLASS(); i++) {
		if (TK_GET_CLASS(res.res_un.obtain_res.granted, i) != TK_NULLSET) {
			cp->target[i] = res.res_un.obtain_res.data[i];
		}
	}
	*already = res.res_un.obtain_res.already;
}

/*
 * client_revoke - callout from token client module
 */
static void
client_revoke(void *o, tk_set_t revoke)
{
	struct test2_req req;
	struct cdata *cp = (struct cdata *)o;
	int i;
	auto tk_set_t tret;
	struct sdata *sp;

	if (AM_S1S(cp->node))
		sp = S1_P(cp->node);

	/*
	 * if our existance token was recalled, then shut down entire object
	 * This includes sending back all tokens we might have cached
	 */
	if (TK_GET_CLASS(revoke, S1_NCLASS()) != TK_NULLSET) {
		tkc_returning(cp->cs, S1_ALLTK(), &tret, 0);
		revoke |= tret;
	}

	if (AM_S1S(cp->node)) {
		/* we are server */
		tks_return(sp->ss, cp->h, revoke);
	} else {
		/* send return message and data */
		req.req_un.return_req.toreturn = revoke;
		req.op = TEST2_RETURN;
		req.service_id = SERVICE1_ID;
		for (i = 0; i < S1_NCLASS(); i++) {
			if (TK_GET_CLASS(revoke, i) != TK_NULLSET) {
				req.req_un.return_req.data[i] = cp->target[i];
			}
		}

		req.addr.handle = cp->h;
		writembox_copy(ntombox(S1S()), &req, sizeof(req));
	}

	/* inform client token module that all is done */
	tkc_return(cp->cs, revoke, TK_NULLSET);

	if (TK_GET_CLASS(revoke, S1_NCLASS()) != TK_NULLSET)
		tkc_destroy(cp->cs);
}

/*
 * callout when client receives a recall request
 * For service1 a recall means to shut down usage of the object
 */
static void
client_recall(void *o, tk_set_t toreturn, tk_set_t refuse, tk_set_t eh)
{
	struct test2_req req;
	struct cdata *cp = (struct cdata *)o;
	int i;
	auto tk_set_t tret;
	struct sdata *sp;

	if (AM_S1S(cp->node))
		sp = S1_P(cp->node);

	/*
	 * if our existance token was recalled, then shut down entire object
	 * This includes sending back all tokens we might have cached
	 */
	if (TK_GET_CLASS(toreturn, S1_NCLASS()) != TK_NULLSET) {
		tkc_returning(cp->cs, S1_ALLTK(), &tret, 0);
		toreturn |= tret;
	}

	if (AM_S1S(cp->node)) {
		/* we are server */
		tks_return_recall(sp->ss, cp->h, toreturn, refuse, eh);
	} else {
		/* send return message and data */
		req.req_un.return_req.toreturn = toreturn;
		req.req_un.return_req.refuse = refuse;
		req.req_un.return_req.eh = eh;
		req.op = TEST2_RETURN_RECALL;
		req.service_id = SERVICE1_ID;
		for (i = 0; i < S1_NCLASS(); i++) {
			if (TK_GET_CLASS(toreturn, i) != TK_NULLSET) {
				req.req_un.return_req.data[i] = cp->target[i];
			}
		}

		req.addr.handle = cp->h;
		writembox_copy(ntombox(S1S()), &req, sizeof(req));
	}

	/* inform client token module that all is done */
	tkc_return(cp->cs, toreturn, TK_NULLSET);

	if (TK_GET_CLASS(toreturn, S1_NCLASS()) != TK_NULLSET)
		tkc_destroy(cp->cs);
}

/*
 * get client handle from within token module - used for TRACING only
 */
tks_ch_t
getch_t(void *o)
{
	struct cdata *cp = (struct cdata *)o;

	assert(cp);
	return(cp->h);
}


/*
 * callout function that is called from the token server module when a
 * token (set) needs to be revoked. It sends a message to the appropriate
 * client. This sends a 1 way message
 */
/* ARGSUSED */
static void
server_revoke(void *o, tks_ch_t h, tk_set_t r)
{
	struct test2_req req;
	struct sdata *sp = (struct sdata *)o;

	if (AM_S1S(htonode(h))) {
		tkc_revoke(sp->cp->cs, r);
		return;
	}

	req.req_un.revoke_req.torevoke = r;
	req.op = TEST2_REVOKE;
	req.service_id = SERVICE1_ID;

	writembox_copy(ntombox(htonode(h)), &req, sizeof(req));
}

/*
 * callout function to start recall process - called as the result of
 * a tks_recall operation
 */
static void
server_recall(void *o, tks_ch_t h, tk_set_t r, int flag)
{
	struct test2_req req;
	struct sdata *sp = (struct sdata *)o;

	if (AM_S1S(htonode(h))) {
		tkc_recall(sp->cp->cs, r, flag);
		return;
	}

	req.req_un.recall_req.torecall = r;
	req.req_un.recall_req.flag = flag;
	req.op = TEST2_RECALL;
	req.service_id = SERVICE1_ID;

	writembox_copy(ntombox(htonode(h)), &req, sizeof(req));
}

/*
 * callout function - called when a token has been recalled and can now be
 * reclaimed/destroyed
 */
/* ARGSUSED */
static void
service1_recalled(void *o, tk_set_t r, tk_set_t refused)
{
	struct sdata *sp = (struct sdata *)o;

	assert(AM_S1S(sp->cp->node));
	assert(refused == TK_NULLSET);
	usvsema(sp->sync);
}

/*
 * server_return - called by the server message handler when a TEST2_RETURN or
 * TEST2_RETURN_RECALL
 * message is received. The data is updated and the token(s) returned.
 */
static void
server_return(struct sdata *sp, struct test2_req *req)
{
	tk_set_t r;
	int i;

	r = req->req_un.return_req.toreturn;

	/* 
	 * update any modified data
	 * We shouldn't need any locks since the caller still has the
	 * token for WR or RD, noone should be able to change it
	 */
	for (i = 0; i < S1_NCLASS(); i++) {
		if (TK_IS_IN_SET(r, TK_MAKE(i, TK_WRITE))) {
			sp->cp->target[i] = req->req_un.return_req.data[i];
		} else if (TK_IS_IN_SET(r, TK_MAKE(i, TK_READ))) {
			if (sp->cp->target[i] != req->req_un.return_req.data[i]) {
				printf("server revoked a read token and got back different data.\n");
				printf("\tserver data:%d client data %d\n",
					sp->cp->target[i],
					req->req_un.return_req.data[i]);
				abort();
			}
		}
	}

	/* return set of tokens */
	if (req->op == TEST2_RETURN)
		tks_return(sp->ss, req->addr.handle, r);
	else
		tks_return_recall(sp->ss, req->addr.handle, r,
			req->req_un.return_req.refuse,
			req->req_un.return_req.eh);
}

/*
 * this thread waits for any out-of-the-blue messages
 * Both client and server.
 * 'node' is which node we are.
 */
static void
service1_wait(struct test2_req *req, int node)
{
	struct cdata *cp;	/* client side data */
	struct sdata *sp;	/* server side data */
	struct test2_res res;
	tk_set_t toreturn;
	int i;

	switch (req->op) {
	/* client messages */
	case TEST2_REVOKE:
		cp = (struct cdata *)S1_P(node);
		assert(!AM_S1S(node));
		tkc_revoke(cp->cs, req->req_un.revoke_req.torevoke);
		break;
	case TEST2_RECALL:
		cp = (struct cdata *)S1_P(node);
		assert(!AM_S1S(node));
		tkc_recall(cp->cs, req->req_un.recall_req.torecall,
			req->req_un.recall_req.flag);
		break;
	/* server messages */
	case TEST2_OBTAIN:
		sp = (struct sdata *)S1_P(node);
		assert(AM_S1S(node));
		toreturn = req->req_un.obtain_req.toreturn;
		if (toreturn != TK_NULLSET) {
			/*
			 * if returning write token - update data
			 */
			for (i = 0; i < S1_NCLASS(); i++) {
				if (TK_IS_IN_SET(toreturn, TK_MAKE(i, TK_WRITE)))
					sp->cp->target[i] = req->req_un.obtain_req.data[i];
			}
			tks_return(sp->ss, req->addr.handle, toreturn);
		}

		tks_obtain(sp->ss, req->addr.handle,
				req->req_un.obtain_req.obtain,
				&res.res_un.obtain_res.granted,
				&res.res_un.obtain_res.already);
		for (i = 0; i < S1_NCLASS(); i++) {
			if (TK_GET_CLASS(res.res_un.obtain_res.granted, i) != TK_NULLSET)
				res.res_un.obtain_res.data[i] =
							sp->cp->target[i];
		}
		res.op = TEST2_OBTAIN;
		writembox_copy((struct mbox *)req->addr.return_addr,
						&res, sizeof(res));
		break;
	case TEST2_RETURN_RECALL:
	case TEST2_RETURN:
		sp = (struct sdata *)S1_P(node);
		assert(AM_S1S(node));
		/* 1-way message in response to a revoke/recall */
		server_return(sp, req);
		break;
	case TEST2_GEN:
		{
		auto tk_set_t granted, already;

		/* client wants to play at 'gen' - wait till server is ready */
		sp = (struct sdata *)S1_P(node);
		assert(AM_S1S(node));
		ussetlock(sp->cp->lock);
		if (sp->gen > req->req_un.gen_req.gen) {
			abort();
		} else if (sp->gen < req->req_un.gen_req.gen) {
			/* wait */
			assert(sp->nwaitgen >= 0);
			sp->nwaitgen++;
			usunsetlock(sp->cp->lock);
			uspsema(sp->gensync);
		} else
			usunsetlock(sp->cp->lock);
		/* hack - increment # clients */
		ussetlock(sp->cp->lock);
		sp->ncl++;
		usunsetlock(sp->cp->lock);

		/* all set - respond with existance token */
		tks_obtain(sp->ss, req->addr.handle,
				TK_MAKE(S1_NCLASS(), TK_READ),
				&granted,
				&already);
		assert(already == TK_NULLSET);
		assert(granted == TK_MAKE(S1_NCLASS(), TK_READ));
		res.op = TEST2_GEN;
		res.res_un.gen_res.existance = granted;
		writembox_copy((struct mbox *)req->addr.return_addr,
						&res, sizeof(res));
		break;
		}
	default:
		fprintf(stderr, "service1-wait:Unknown msg op:%d\n", req->op);
		abort();
		break;
	}
}

/* 
 * rpc - 
 */
static int
srpc(struct mbox *to, tks_ch_t h, void *req, int reqlen, void *res, int reslen)
{
	struct mbox *mb;

	mb = allocmbox();

	((struct scheme *)req)->handle = h;	/* our handle */
	((struct scheme *)req)->return_addr = mb;/* where to return message to */
	writembox(to, req, reqlen);

	/* wait for response */
	readmbox_copy(mb, res, reslen);
	freembox(mb);

	return 0;
}