Diff | Source/Git

Download diff

	4b825dc642cb6eb9a060e54bf8d69288fbee4904		cfad47cfa334b206c65f22086bcc5d63e6f70944
		1	#ifdef RCSID
		2	static char RCSid[] =
		3	"$Header: d:/cvsroot/tads/TADS2/TOK.C,v 1.3 1999/07/11 00:46:30 MJRoberts Exp $";
		4	#endif
		5
		6	/*
		7	* Copyright (c) 1987, 1988 by Michael J. Roberts. All Rights Reserved.
		8	*
		9	* Please see the accompanying license file, LICENSE.TXT, for information
		10	* on using and copying this software.
		11	*/
		12	/*
		13	Name
		14	tok - tokenizer
		15	Function
		16	Tokenizes an input stream
		17	Notes
		18	None
		19	Modified
		20	04/11/99 CNebel - Enforce identifier length limits.
		21	08/14/91 MJRoberts - created
		22	*/
		23
		24	#include "os.h"
		25	#include "std.h"
		26	#include "mch.h"
		27	#include "mcm.h"
		28	#include "tok.h"
		29	#include "lin.h"
		30	#include "linf.h"
		31	#include "dbg.h"
		32	#include <ctype.h>
		33	#include <string.h>
		34	#include <stdlib.h>
		35
		36
		37	/* special temporary buffers for <<expr>> macro expansion */
		38	static char tokmac1[] = ",say((";
		39	static char tokmac1s[] = "(say((";
		40	static char tokmac2[] = "),nil),\"";
		41	static char tokmac3[] = "),nil))";
		42	static char tokmac4[] = ")";
		43
		44	/* forward definition of static functions */
		45	static int tokdfhsh(char *sym, int len);
		46
		47
		48	/* find a #define symbol */
		49	static tokdfdef tok_find_define(tokcxdef ctx, char *sym, int len)
		50	{
		51	int hsh;
		52	tokdfdef *df;
		53
		54	/* find the appropriate chain the hash table */
		55	hsh = tokdfhsh(sym, len);
		56
		57	/* search the chain for this symbol */
		58	for (df = ctx->tokcxdf[hsh] ; df ; df = df->nxt)
		59	{
		60	/* if this one matches, return it */
		61	if (df->len == len && !memcmp(df->nm, sym, (size_t)len))
		62	{
		63	/* fix it up if it's the special __FILE__ or __LINE__ symbol */
		64	if (len == 8)
		65	{
		66	if (!memcmp(sym, "__FILE__", (size_t)8))
		67	{
		68	size_t len;
		69
		70	/*
		71	* put in the opening single quote, since we want
		72	* the expanded result to be a string
		73	*/
		74	df->expan[0] = '\'';
		75
		76	/* get the name */
		77	linnam(ctx->tokcxlin, df->expan+1);
		78
		79	/* get the length, and add the closing quote */
		80	len = strlen(df->expan);
		81	df->expan[len] = '\'';
		82
		83	/*
		84	* set the length of the expansion, including the
		85	* quotes (the first quote was measured in the
		86	* length originally, but the second quote hasn't
		87	* been counted yet, so add one to our original
		88	* length)
		89	*/
		90	df->explen = (int)len + 1;
		91
		92	/* if the expansion is too long, it's an error */
		93	if (df->explen >= OSFNMAX)
		94	errsig(ctx->tokcxerr, ERR_LONG_FILE_MACRO);
		95	}
		96	else if (!memcmp(sym, "__LINE__", (size_t)8))
		97	{
		98	ulong l;
		99
		100	/* get the line number */
		101	l = linlnum(ctx->tokcxlin);
		102
		103	/* convert it to a textual format for the expansion */
		104	sprintf(df->expan, "%lu", l);
		105
		106	/* set the expanded value's length */
		107	df->explen = strlen(df->expan);
		108
		109	/* make sure the expansion isn't too long */
		110	if (df->explen >= 40)
		111	errsig(ctx->tokcxerr, ERR_LONG_LINE_MACRO);
		112	}
		113	}
		114
		115	/* return it */
		116	return df;
		117	}
		118	}
		119
		120	/* didn't find anything */
		121	return 0;
		122	}
		123
		124	/*
		125	* Write preprocessor state to a file
		126	*/
		127	void tok_write_defines(tokcxdef ctx, osfildef fp, errcxdef *ec)
		128	{
		129	int i;
		130	tokdfdef **dfp;
		131	tokdfdef *df;
		132	char buf[4];
		133
		134	/* write each element of the hash chains */
		135	for (i = TOKDFHSHSIZ, dfp = ctx->tokcxdf ; i ; ++dfp, --i)
		136	{
		137	/* write each entry in this hash chain */
		138	for (df = *dfp ; df ; df = df->nxt)
		139	{
		140	/* write this entry */
		141	oswp2(buf, df->len);
		142	oswp2(buf + 2, df->explen);
		143	if (osfwb(fp, buf, 4)
		144	\|\| osfwb(fp, df->nm, df->len)
		145	\|\| (df->explen != 0 && osfwb(fp, df->expan, df->explen)))
		146	errsig(ec, ERR_WRTGAM);
		147	}
		148
		149	/* write a zero-length entry to indicate the end of this chain */
		150	oswp2(buf, 0);
		151	if (osfwb(fp, buf, 4)) errsig(ec, ERR_WRTGAM);
		152	}
		153	}
		154
		155	/*
		156	* Read preprocessor state from a file
		157	*/
		158	void tok_read_defines(tokcxdef ctx, osfildef fp, errcxdef *ec)
		159	{
		160	int i;
		161	tokdfdef **dfp;
		162	tokdfdef *df;
		163	char buf[4];
		164
		165	/* write each element of the hash chains */
		166	for (i = TOKDFHSHSIZ, dfp = ctx->tokcxdf ; i ; ++dfp, --i)
		167	{
		168	/* read this hash chain */
		169	for (;;)
		170	{
		171	/* read the next entry's header, and stop if this is the end */
		172	if (osfrb(fp, buf, 4)) errsig(ec, ERR_RDGAM);
		173	if (osrp2(buf) == 0) break;
		174
		175	/* set up a new symbol of the appropriate size */
		176	df = (tokdfdef *)mchalo(ec,
		177	(ushort)(sizeof(tokdfdef) + osrp2(buf)
		178	+ osrp2(buf+2) - 1),
		179	"tok_read_defines");
		180	df->explen = osrp2(buf+2);
		181	df->nm = df->expan + df->explen;
		182	df->len = osrp2(buf);
		183
		184	/* read the rest of the symbol */
		185	if (osfrb(fp, df->nm, df->len)
		186	\|\| (df->explen != 0 && osfrb(fp, df->expan, df->explen)))
		187	errsig(ec, ERR_RDGAM);
		188
		189	/*
		190	* If a symbol with this name already exists in the table,
		191	* discard the new one -- the symbols defined by -D and the
		192	* current set of built-in symbols takes precedence over the
		193	* set loaded from the file.
		194	*/
		195	if (tok_find_define(ctx, df->nm, df->len))
		196	{
		197	/* simply discard this symbol */
		198	mchfre(df);
		199	}
		200	else
		201	{
		202	/* link it into this hash chain */
		203	df->nxt = *dfp;
		204	*dfp = df;
		205	}
		206	}
		207	}
		208	}
		209
		210
		211
		212	/* compute a #define symbol's hash value */
		213	static int tokdfhsh(char *sym, int len)
		214	{
		215	uint hsh;
		216
		217	for (hsh = 0 ; len ; ++sym, --len)
		218	hsh = (hsh + *sym) & TOKDFHSHMASK;
		219	return hsh;
		220	}
		221
		222	/* convert a #define symbol to lower case if folding case */
		223	static char tok_casefold_defsym(tokcxdef ctx, char *outbuf,
		224	char *src, int len)
		225	{
		226	if (ctx->tokcxflg & TOKCXCASEFOLD)
		227	{
		228	char *dst;
		229	int rem;
		230
		231	/* make a lower-case copy of the symbol */
		232	rem = (len > TOKNAMMAX ? TOKNAMMAX : len);
		233	for (dst = outbuf ; rem > 0 ; ++dst, ++src, --rem)
		234	dst = (isupper((uchar)src) ? tolower((uchar)src) : src);
		235
		236	/* use the lower-case copy instead of the original */
		237	return outbuf;
		238	}
		239	else
		240	{
		241	/* return the original unchanged */
		242	return src;
		243	}
		244	}
		245
		246	/*
		247	* convert a token to lower-case if we're folding case
		248	*/
		249	void tok_case_fold(tokcxdef ctx, tokdef tok)
		250	{
		251	/* if we're in case-insensitive mode, convert the token to lower-case */
		252	if (ctx->tokcxflg & TOKCXCASEFOLD)
		253	{
		254	char *p;
		255	int len;
		256
		257	/* convert each character in the token to lower-case */
		258	for (p = tok->toknam, len = tok->toklen ; len != 0 ; ++p, --len)
		259	{
		260	/* if this character is upper-case, convert it to lower-case */
		261	if (isupper((uchar)*p))
		262	p = tolower((uchar)p);
		263	}
		264	}
		265	}
		266
		267	/* add a symbol to the #define symbol table, folding case if necessary */
		268	void tok_add_define_cvtcase(tokcxdef ctx, char sym, int len,
		269	char *expan, int explen)
		270	{
		271	char mysym[TOKNAMMAX];
		272
		273	/* convert to lower-case if necessary */
		274	sym = tok_casefold_defsym(ctx, mysym, sym, len);
		275
		276	/* add the symbol */
		277	tok_add_define(ctx, sym, len, expan, explen);
		278	}
		279
		280	/* add a symbol to the #define symbol table */
		281	void tok_add_define(tokcxdef ctx, char sym, int len,
		282	char *expan, int explen)
		283	{
		284	int hsh;
		285	tokdfdef *df;
		286
		287	/* if it's already defined, ignore it */
		288	if (tok_find_define(ctx, sym, len))
		289	return;
		290
		291	/* find the appropriate entry in the hash table */
		292	hsh = tokdfhsh(sym, len);
		293
		294	/* allocate space for the symbol */
		295	df = (tokdfdef *)mchalo(ctx->tokcxerr,
		296	(ushort)(sizeof(tokdfdef) + len + explen - 1),
		297	"tok_add_define");
		298
		299	/* set up the new symbol */
		300	df->nm = df->expan + explen;
		301	df->len = len;
		302	df->explen = explen;
		303	memcpy(df->expan, expan, explen);
		304	memcpy(df->nm, sym, len);
		305
		306	/* link it into the hash chain */
		307	df->nxt = ctx->tokcxdf[hsh];
		308	ctx->tokcxdf[hsh] = df;
		309	}
		310
		311	/* add a #define symbol with a numeric value */
		312	void tok_add_define_num_cvtcase(tokcxdef ctx, char sym, int len, int num)
		313	{
		314	char buf[20];
		315
		316	/* convert the value to a string */
		317	sprintf(buf, "%d", num);
		318
		319	/* add the text value */
		320	tok_add_define_cvtcase(ctx, sym, len, buf, strlen(buf));
		321	}
		322
		323	/* undefine a #define symbol */
		324	void tok_del_define(tokcxdef ctx, char sym, int len)
		325	{
		326	int hsh;
		327	tokdfdef *df;
		328	tokdfdef *prv;
		329
		330	/* find the appropriate chain the hash table */
		331	hsh = tokdfhsh(sym, len);
		332
		333	/* search the chain for this symbol */
		334	for (prv = 0, df = ctx->tokcxdf[hsh] ; df ; prv = df, df = df->nxt)
		335	{
		336	/* if this one matches, delete it */
		337	if (df->len == len && !memcmp(df->nm, sym, (size_t)len))
		338	{
		339	/* unlink it from the chain */
		340	if (prv)
		341	prv->nxt = df->nxt;
		342	else
		343	ctx->tokcxdf[hsh] = df->nxt;
		344
		345	/* delete this symbol, and we're done */
		346	mchfre(df);
		347	break;
		348	}
		349	}
		350	}
		351
		352	/* scan a #define symbol to see how long it is */
		353	static int tok_scan_defsym(tokcxdef ctx, char p, int len)
		354	{
		355	int symlen;
		356
		357	/* make sure it's a valid symbol */
		358	if (!(isalpha((uchar)p) \|\| p == '_' \|\| *p == '$'))
		359	{
		360	errlog(ctx->tokcxerr, ERR_REQSYM);
		361	return 0;
		362	}
		363
		364	/* count characters as long as we have valid symbol characters */
		365	for (symlen = 0 ; len && TOKISSYM(*p) ; ++p, --len, ++symlen) ;
		366	return symlen;
		367	}
		368
		369	/* process a #define */
		370	static void tokdefine(tokcxdef ctx, char p, int len)
		371	{
		372	char *sym;
		373	int symlen;
		374	char *expan;
		375	char mysym[TOKNAMMAX];
		376
		377	/* get the symbol */
		378	sym = p;
		379	if (!(symlen = tok_scan_defsym(ctx, p, len)))
		380	return;
		381
		382	/* if it's already in the table, log an error */
		383	if (tok_find_define(ctx, sym, symlen))
		384	{
		385	errlog(ctx->tokcxerr, ERR_DEFREDEF);
		386	return;
		387	}
		388
		389	/* skip whitespace following the symbol */
		390	expan = sym + symlen;
		391	len -= symlen;
		392	while (len && t_isspace(*expan)) --len, ++expan;
		393
		394	/* if we're folding case, convert the symbol to lower case */
		395	sym = tok_casefold_defsym(ctx, mysym, sym, symlen);
		396
		397	/* define the symbol */
		398	tok_add_define(ctx, sym, symlen, expan, len);
		399	}
		400
		401	/*
		402	* Update the #if status for the current nesting. Any enclosing
		403	* negative #if will override everything inside, so we need to look
		404	* through the nesting from the outside in until we either determine
		405	* that everything is affirmative or we find a negative anywhere in the
		406	* nesting.
		407	*/
		408	static void tok_update_if_stat(tokcxdef *ctx)
		409	{
		410	int i;
		411
		412	/* look through nesting from the outermost level */
		413	for (i = 0 ; i < ctx->tokcxifcnt ; ++i)
		414	{
		415	/* assume this level will apply to everything inside */
		416	ctx->tokcxifcur = ctx->tokcxif[i];
		417
		418	/* if this level is off, everything inside is off */
		419	switch (ctx->tokcxif[i])
		420	{
		421	case TOKIF_IF_NO:
		422	case TOKIF_ELSE_NO:
		423	/*
		424	* this level is off, hence everything inside is off -- stop
		425	* here with the current (negative) determination
		426	*/
		427	return;
		428
		429	default:
		430	/* so far we're in the "on" section, so keep looking */
		431	break;
		432	}
		433	}
		434	}
		435
		436	/* process an #ifdef or a #ifndef */
		437	static void tok_ifdef_ifndef(tokcxdef ctx, char p, int len, int is_ifdef)
		438	{
		439	int symlen;
		440	char *sym;
		441	int stat;
		442	int found;
		443	char mysym[TOKNAMMAX];
		444
		445	/* get the symbol */
		446	sym = p;
		447	if (!(symlen = tok_scan_defsym(ctx, p, len)))
		448	return;
		449
		450	/* if we're folding case, convert the symbol to lower case */
		451	sym = tok_casefold_defsym(ctx, mysym, sym, symlen);
		452
		453	/* see if we can find it in the table, and set the status accordingly */
		454	found = (tok_find_define(ctx, sym, symlen) != 0);
		455
		456	/* invert the test if this is an ifndef */
		457	if (!is_ifdef) found = !found;
		458
		459	/* set the #if status accordingly */
		460	if (found)
		461	stat = TOKIF_IF_YES;
		462	else
		463	stat = TOKIF_IF_NO;
		464	ctx->tokcxif[ctx->tokcxifcnt] = stat;
		465
		466	/* allocate a new #if level (making sure we have room) */
		467	if (ctx->tokcxifcnt >= TOKIFNEST)
		468	{
		469	errlog(ctx->tokcxerr, ERR_MANYPIF);
		470	return;
		471	}
		472	ctx->tokcxifcnt++;
		473
		474	/* update the current status */
		475	tok_update_if_stat(ctx);
		476	}
		477
		478	/* process a #error */
		479	static void tok_p_error(tokcxdef ctx, char p, int len)
		480	{
		481	errlog1(ctx->tokcxerr, ERR_P_ERROR,
		482	ERRTSTR, errstr(ctx->tokcxerr, p, len));
		483	}
		484
		485	/* process a #ifdef */
		486	static void tokifdef(tokcxdef ctx, char p, int len)
		487	{
		488	tok_ifdef_ifndef(ctx, p, len, TRUE);
		489	}
		490
		491	/* process a #ifndef */
		492	static void tokifndef(tokcxdef ctx, char p, int len)
		493	{
		494	tok_ifdef_ifndef(ctx, p, len, FALSE);
		495	}
		496
		497	/* process a #if */
		498	static void tokif(tokcxdef ctx, char p, int len)
		499	{
		500	errsig(ctx->tokcxerr, ERR_PIF_NA);
		501	}
		502
		503	/* process a #elif */
		504	static void tokelif(tokcxdef ctx, char p, int len)
		505	{
		506	errsig(ctx->tokcxerr, ERR_PELIF_NA);
		507	}
		508
		509	/* process a #else */
		510	static void tokelse(tokcxdef ctx, char p, int len)
		511	{
		512	int cnt;
		513
		514	/* if we're not expecting #else, it's an error */
		515	cnt = ctx->tokcxifcnt;
		516	if (cnt == 0 \|\| ctx->tokcxif[cnt-1] == TOKIF_ELSE_YES
		517	\|\| ctx->tokcxif[cnt-1] == TOKIF_ELSE_NO)
		518	{
		519	errlog(ctx->tokcxerr, ERR_BADPELSE);
		520	return;
		521	}
		522
		523	/* switch to the appropriate #else state (opposite the #if state) */
		524	if (ctx->tokcxif[cnt-1] == TOKIF_IF_YES)
		525	ctx->tokcxif[cnt-1] = TOKIF_ELSE_NO;
		526	else
		527	ctx->tokcxif[cnt-1] = TOKIF_ELSE_YES;
		528
		529	/* update the current status */
		530	tok_update_if_stat(ctx);
		531	}
		532
		533	/* process a #endif */
		534	static void tokendif(tokcxdef ctx, char p, int len)
		535	{
		536	/* if we're not expecting #endif, it's an error */
		537	if (ctx->tokcxifcnt == 0)
		538	{
		539	errlog(ctx->tokcxerr, ERR_BADENDIF);
		540	return;
		541	}
		542
		543	/* remove the #if level */
		544	ctx->tokcxifcnt--;
		545
		546	/* update the current status */
		547	tok_update_if_stat(ctx);
		548	}
		549
		550	/* process a #undef */
		551	static void tokundef(tokcxdef ctx, char p, int len)
		552	{
		553	char *sym;
		554	int symlen;
		555	char mysym[TOKNAMMAX];
		556
		557	/* get the symbol */
		558	sym = p;
		559	if (!(symlen = tok_scan_defsym(ctx, p, len)))
		560	return;
		561
		562	/* if we're folding case, convert the symbol to lower case */
		563	sym = tok_casefold_defsym(ctx, mysym, sym, symlen);
		564
		565	/* if it's not defined, log a warning */
		566	if (!tok_find_define(ctx, sym, symlen))
		567	{
		568	errlog(ctx->tokcxerr, ERR_PUNDEF);
		569	return;
		570	}
		571
		572	/* undefine the symbol */
		573	tok_del_define(ctx, sym, symlen);
		574	}
		575
		576	/* process a #pragma directive */
		577	static void tokpragma(tokcxdef ctx, char p, int len)
		578	{
		579	/* ignore empty pragmas */
		580	if (len == 0)
		581	{
		582	errlog(ctx->tokcxerr, ERR_PRAGMA);
		583	return;
		584	}
		585
		586	/* see what we have */
		587	if (len > 1 && (p == 'c' \|\| p == 'C')
		588	&& ((p+1) == '+' \|\| (p+1) == '-') \|\| t_isspace(*(p+1)))
		589	{
		590	/* skip spaces after the 'C', if any */
		591	for (++p, --len ; len && t_isspace(*p) ; ++p, --len) ;
		592
		593	/* look for the + or - flag */
		594	if (len && *p == '+')
		595	ctx->tokcxflg \|= TOKCXFCMODE;
		596	else if (len && *p == '-')
		597	ctx->tokcxflg &= ~TOKCXFCMODE;
		598	else
		599	{
		600	errlog(ctx->tokcxerr, ERR_PRAGMA);
		601	return;
		602	}
		603	}
		604	else
		605	{
		606	errlog(ctx->tokcxerr, ERR_PRAGMA);
		607	}
		608	}
		609
		610	/* process a #include directive */
		611	static void tokinclude(tokcxdef ctx, char p, int len)
		612	{
		613	linfdef *child;
		614	tokpdef *path;
		615	char *fname;
		616	int match;
		617	int flen;
		618	linfdef *lin;
		619	char *q;
		620	size_t flen2;
		621
		622	/* find the filename portion */
		623	fname = p + 1; /* remember start of filename */
		624	path = ctx->tokcxinc; /* start with first path entry */
		625
		626	if (!len)
		627	{
		628	errlog(ctx->tokcxerr, ERR_INCNOFN);
		629	return;
		630	}
		631
		632	switch(*p)
		633	{
		634	case '<':
		635	match = '>';
		636	if (path && path->tokpnxt) path = path->tokpnxt; /* skip 1st path */
		637	goto find_matching_delim;
		638
		639	case '"':
		640	match = '"';
		641
		642	find_matching_delim:
		643	for (++p, --len ; len && *p != match ; --len, ++p) ;
		644	if (len == 0 \|\| *p != match) errlog(ctx->tokcxerr, ERR_INCMTCH);
		645	break;
		646
		647	default:
		648	errlog(ctx->tokcxerr, ERR_INCSYN);
		649	return;
		650	}
		651
		652	flen = p - fname; /* compute length of filename */
		653	for (q = p, flen2 = 0 ;
		654	q > fname && (q-1) != OSPATHCHAR && !strchr(OSPATHALT, (q-1)) ;
		655	--q, ++flen2) ;
		656
		657	/* check to see if this file has already been included */
		658	for (lin = ctx->tokcxhdr ; lin ; lin = (linfdef *)lin->linflin.linnxt)
		659	{
		660	char *p = lin->linfnam;
		661
		662	p += strlen(p);
		663
		664	while (p > lin->linfnam && *(p-1) != OSPATHCHAR
		665	&& !strchr(OSPATHALT, *(p-1)))
		666	--p;
		667	if (strlen(p) == flen2
		668	&& !memicmp(p, q, flen2))
		669	{
		670	errlog1(ctx->tokcxerr, ERR_INCRPT, ERRTSTR,
		671	errstr(ctx->tokcxerr, fname, flen));
		672	return;
		673	}
		674	}
		675
		676	/* initialize the line source */
		677	child = linfini(ctx->tokcxmem, ctx->tokcxerr, fname, flen, path, TRUE,
		678	(ctx->tokcxflg & TOKCXFLIN2) != 0);
		679
		680	/* if not found, signal an error */
		681	if (!child) errsig1(ctx->tokcxerr, ERR_INCSEAR,
		682	ERRTSTR, errstr(ctx->tokcxerr, fname, flen));
		683
		684	/* link into tokenizer list of line records */
		685	child->linflin.linnxt = (lindef *)ctx->tokcxhdr;
		686	ctx->tokcxhdr = child;
		687
		688	/* if we're tracking sources for debugging, add into the chain */
		689	if (ctx->tokcxdbg)
		690	{
		691	ctx->tokcxdbg->dbgcxlin = &child->linflin;
		692	child->linflin.linid = ctx->tokcxdbg->dbgcxfid++;
		693	}
		694
		695	/* remember my C-mode setting */
		696	if (ctx->tokcxflg & TOKCXFCMODE)
		697	ctx->tokcxlin->linflg \|= LINFCMODE;
		698	else
		699	ctx->tokcxlin->linflg &= ~LINFCMODE;
		700
		701	child->linflin.linpar = ctx->tokcxlin; /* remember parent line source */
		702	ctx->tokcxlin = &child->linflin; /* make the child the current source */
		703	}
		704
		705	/* get a new line from line source, processing '#' directives */
		706	static int tokgetlin(tokcxdef *ctx, int dopound)
		707	{
		708	for (;;)
		709	{
		710	if (linget(ctx->tokcxlin))
		711	{
		712	/* at eof in current source; resume parent if there is one */
		713	if (ctx->tokcxlin->linpar)
		714	{
		715	lindef *parent;
		716
		717	parent = ctx->tokcxlin->linpar; /* remember parent */
		718	lincls(ctx->tokcxlin); /* close included file */
		719	if (!ctx->tokcxdbg) /* if no debug context... */
		720	mchfre(ctx->tokcxlin); /* free line source */
		721	ctx->tokcxlin = parent; /* reset to parent line source */
		722	if (parent->linflg & LINFCMODE)
		723	ctx->tokcxflg \|= TOKCXFCMODE;
		724	else
		725	ctx->tokcxflg &= ~TOKCXFCMODE;
		726	continue; /* back for another attempt */
		727	}
		728	else
		729	{
		730	/* check for outstanding #if/#ifdef */
		731	if (ctx->tokcxifcnt)
		732	errlog(ctx->tokcxerr, ERR_NOENDIF);
		733
		734	/* return end-of-file indication */
		735	return TRUE;
		736	}
		737	}
		738
		739	/* if this is a multi-segment line, copy it into our own buffer */
		740	if (ctx->tokcxlin->linflg & LINFMORE)
		741	{
		742	char *p;
		743	uint rem;
		744	int done;
		745
		746	if (!ctx->tokcxbuf)
		747	{
		748	/* allocate 1k as a default buffer */
		749	ctx->tokcxbuf = (char *)mchalo(ctx->tokcxerr, (ushort)1024,
		750	"tok");
		751	ctx->tokcxbsz = 1024;
		752	}
		753	ctx->tokcxlen = 0;
		754
		755	for (done = FALSE, p = ctx->tokcxbuf, rem = ctx->tokcxbsz ;
		756	!done ; )
		757	{
		758	size_t len = ctx->tokcxlin->linlen;
		759
		760	/* add the current segment's length into line length */
		761	ctx->tokcxlen += len;
		762
		763	/* we're done after this piece if the last fetch was all */
		764	done = !(ctx->tokcxlin->linflg & LINFMORE);
		765	if (len + 1 > rem)
		766	{
		767	char *newp;
		768
		769	/* increase the size of the buffer */
		770	if (ctx->tokcxbsz > (unsigned)0x8000)
		771	errsig(ctx->tokcxerr, ERR_LONGLIN);
		772	rem += 4096;
		773	ctx->tokcxbsz += 4096;
		774
		775	/* allocate a new buffer and copy line into it */
		776	newp = (char *)mchalo(ctx->tokcxerr, ctx->tokcxbsz, "tok");
		777	memcpy(newp, ctx->tokcxbuf, (size_t)(p - ctx->tokcxbuf));
		778
		779	/* free the original buffer, and use the new one */
		780	p = (p - ctx->tokcxbuf) + newp;
		781	mchfre(ctx->tokcxbuf);
		782	ctx->tokcxbuf = newp;
		783	}
		784
		785	/* add the line to the buffer */
		786	memcpy(p, ctx->tokcxlin->linbuf, len);
		787	p += len;
		788	rem -= len;
		789
		790	/* get the next piece of the line if there is one */
		791	if (!done)
		792	{
		793	if (linget(ctx->tokcxlin)) break;
		794	}
		795	}
		796
		797	/* null-terminate the buffer, and use it for input */
		798	*p = '\0';
		799	ctx->tokcxptr = ctx->tokcxbuf;
		800	}
		801	else
		802	{
		803	ctx->tokcxptr = ctx->tokcxlin->linbuf;
		804	ctx->tokcxlen = ctx->tokcxlin->linlen;
		805	}
		806
		807	/* check for preprocessor directives */
		808	if (dopound && ctx->tokcxlen != 0 && ctx->tokcxptr[0] == '#'
		809	&& !(ctx->tokcxlin->linflg & LINFNOINC))
		810	{
		811	char *p;
		812	int len;
		813	static struct
		814	{
		815	char *nm;
		816	int len;
		817	int ok_in_if;
		818	void (fn)(tokcxdef , char *, int);
		819	}
		820	*dirp, dir[] =
		821	{
		822	{ "include", 7, FALSE, tokinclude },
		823	{ "pragma", 6, FALSE, tokpragma },
		824	{ "define", 6, FALSE, tokdefine },
		825	{ "ifdef", 5, TRUE, tokifdef },
		826	{ "ifndef", 6, TRUE, tokifndef },
		827	{ "if", 2, TRUE, tokif },
		828	{ "else", 4, TRUE, tokelse },
		829	{ "elif", 4, TRUE, tokelif },
		830	{ "endif", 5, TRUE, tokendif },
		831	{ "undef", 5, FALSE, tokundef },
		832	{ "error", 5, FALSE, tok_p_error }
		833	};
		834	int i;
		835
		836	/* scan off spaces between '#' and directive */
		837	for (len = ctx->tokcxlen - 1, p = &ctx->tokcxptr[1] ;
		838	len && t_isspace(*p) ; --len, ++p) ;
		839
		840	/* find and process the directive */
		841	for (dirp = dir, i = sizeof(dir)/sizeof(dir[0]) ; i ; --i, ++dirp)
		842	{
		843	/* compare this directive; if it wins, call its function */
		844	if (len >= dirp->len && !memcmp(p, dirp->nm, (size_t)dirp->len)
		845	&& (len == dirp->len \|\| t_isspace(*(p + dirp->len))))
		846	{
		847	int cnt;
		848	int stat;
		849
		850	/*
		851	* if we're not in a #if's false part, or if the
		852	* directive is processed even in #if false parts,
		853	* process the line, otherwise skip it
		854	*/
		855	cnt = ctx->tokcxifcnt;
		856	if (dirp->ok_in_if \|\| cnt == 0
		857	\|\| ((stat = ctx->tokcxifcur) == TOKIF_IF_YES
		858	\|\| stat == TOKIF_ELSE_YES))
		859	{
		860	/* skip whitespace following the directive */
		861	for (p += dirp->len, len -= dirp->len ;
		862	len && t_isspace(*p) ;
		863	--len, ++p) ;
		864
		865	/* invoke the function to process this directive */
		866	(*dirp->fn)(ctx, p, len);
		867	}
		868
		869	/* there's no need to look at more directives */
		870	break;
		871	}
		872	}
		873
		874	/* if we didn't find anything, flag the error */
		875	if (i == 0)
		876	errlog(ctx->tokcxerr, ERR_PRPDIR);
		877
		878	/* ignore this line */
		879	continue;
		880	}
		881	else
		882	{
		883	/*
		884	* Check the #if level. If we're in an #if, and we're to
		885	* ignore lines (because of a false condition or an #else
		886	* part for a true condition), skip this line.
		887	*/
		888	if (ctx->tokcxifcnt != 0)
		889	{
		890	switch(ctx->tokcxifcur)
		891	{
		892	case TOKIF_IF_NO:
		893	case TOKIF_ELSE_NO:
		894	/* ignore this line */
		895	continue;
		896
		897	default:
		898	/* we're in a true part - keep the line */
		899	break;
		900	}
		901	}
		902
		903	ctx->tokcxlin->linflg &= ~LINFDBG; /* no debug record yet */
		904	return(FALSE); /* return the line we found */
		905	}
		906	}
		907	}
		908
		909	/* get the next token, removing it from the input stream */
		910	int toknext(tokcxdef *ctx)
		911	{
		912	char *p;
		913	tokdef *tok = &ctx->tokcxcur;
		914	int len;
		915
		916	/*
		917	* Check for the special case that we pushed an open paren prior to
		918	* a string containing an embedded expression. If this is the case,
		919	* immediately return the string we previously parsed.
		920	*/
		921	if ((ctx->tokcxflg & TOKCXF_EMBED_PAREN_PRE) != 0)
		922	{
		923	/*
		924	* convert the token to a string - note that the offset
		925	* information for the string is already in the current token
		926	* structure, since we set everything up for it on the previous
		927	* call where we actually parsed the beginning of the string
		928	*/
		929	tok->toktyp = TOKTDSTRING;
		930
		931	/* clear the special flag - we've now consumed the pushed string */
		932	ctx->tokcxflg &= ~TOKCXF_EMBED_PAREN_PRE;
		933
		934	/* immediately return the string */
		935	return tok->toktyp;
		936	}
		937
		938	/* set up at the current scanning position */
		939	p = ctx->tokcxptr;
		940	len = ctx->tokcxlen;
		941
		942	/* scan off whitespace and comments until we find something */
		943	do
		944	{
		945	skipblanks:
		946	/* if there's nothing on this line, get the next one */
		947	if (len == 0)
		948	{
		949	/* if we're in a macro expansion, continue after it */
		950	if (ctx->tokcxmlvl)
		951	{
		952	ctx->tokcxmlvl--;
		953	p = ctx->tokcxmsav[ctx->tokcxmlvl];
		954	len = ctx->tokcxmsvl[ctx->tokcxmlvl];
		955	}
		956	else
		957	{
		958	if (tokgetlin(ctx, TRUE))
		959	{
		960	tok->toktyp = TOKTEOF;
		961	goto done;
		962	}
		963	p = ctx->tokcxptr;
		964	len = ctx->tokcxlen;
		965	}
		966	}
		967	while (len && t_isspace(p)) ++p, --len; / scan off whitespace */
		968
		969	/* check for comments, and remove if present */
		970	if (len >= 2 && p == '/' && (p+1) == '/')
		971	len = 0;
		972	else if (len >= 2 && p == '/' && (p+1) == '*')
		973	{
		974	while (len < 2 \|\| p != '' \|\| *(p+1) != '/')
		975	{
		976	if (len != 0)
		977	++p, --len;
		978
		979	if (len == 0)
		980	{
		981	if (ctx->tokcxmlvl != 0)
		982	{
		983	ctx->tokcxmlvl--;
		984	p = ctx->tokcxmsav[ctx->tokcxmlvl];
		985	len = ctx->tokcxmsvl[ctx->tokcxmlvl];
		986	}
		987	else
		988	{
		989	if (tokgetlin(ctx, FALSE))
		990	{
		991	ctx->tokcxptr = p;
		992	tok->toktyp = TOKTEOF;
		993	goto done;
		994	}
		995	p = ctx->tokcxptr;
		996	len = ctx->tokcxlen;
		997	}
		998	}
		999	}
		1000	p += 2;
		1001	len -= 2;
		1002	goto skipblanks;
		1003	}
		1004	} while (len == 0);
		1005
		1006	nexttoken:
		1007	if (isalpha((uchar)p) \|\| p == '_' \|\| *p == '$')
		1008	{
		1009	int l;
		1010	int hash;
		1011	char *q;
		1012	toktdef *tab;
		1013	int found = FALSE;
		1014	uchar thischar;
		1015	tokdfdef *df;
		1016
		1017	for (hash = 0, l = 0, q = tok->toknam ;
		1018	len != 0 && TOKISSYM(*p) && l < TOKNAMMAX ;
		1019	(thischar = ((isupper((uchar)*p)
		1020	&& (ctx->tokcxflg & TOKCXCASEFOLD))
		1021	? tolower((uchar)p) : p)),
		1022	(hash = ((hash + thischar) & (TOKHASHSIZE - 1))),
		1023	(*q++ = thischar), ++p, --len, ++l) ;
		1024	*q = '\0';
		1025	if (len != 0 && TOKISSYM(*p))
		1026	{
		1027	while (len != 0 && TOKISSYM(*p)) ++p, --len;
		1028	errlog1(ctx->tokcxerr, ERR_TRUNC, ERRTSTR,
		1029	errstr(ctx->tokcxerr, tok->toknam, tok->toklen));
		1030	}
		1031	tok->toklen = l;
		1032	tok->tokhash = hash;
		1033
		1034	/*
		1035	* check for the special defined() preprocessor operator
		1036	*/
		1037	if (l == 9 && !memcmp(tok->toknam,
		1038	((ctx->tokcxflg & TOKCXCASEFOLD)
		1039	? "__defined" : "__DEFINED"),
		1040	(size_t)9)
		1041	&& len > 2 && p == '(' && TOKISSYM((p+1))
		1042	&& !isdigit((uchar)*(p+1)))
		1043	{
		1044	int symlen;
		1045	char mysym[TOKNAMMAX];
		1046
		1047	/* find the matching ')', allowing only symbolic characters */
		1048	++p, --len;
		1049	for (symlen = 0, q = p ; len && p != ')' && TOKISSYM(p) ;
		1050	++p, --len, ++symlen) ;
		1051
		1052	/* make sure we found the closing paren */
		1053	if (!len \|\| *p != ')')
		1054	errsig(ctx->tokcxerr, ERR_BADISDEF);
		1055	++p, --len;
		1056
		1057	/* if we're folding case, convert the symbol to lower case */
		1058	q = tok_casefold_defsym(ctx, mysym, q, symlen);
		1059
		1060	/* check to see if it's defined */
		1061	tok->toktyp = TOKTNUMBER;
		1062	tok->tokval = (tok_find_define(ctx, q, symlen) != 0);
		1063	goto done;
		1064	}
		1065
		1066	/* substitute the preprocessor #define, if any */
		1067	if ((df = tok_find_define(ctx, tok->toknam, l)) != 0)
		1068	{
		1069	/* save the current parsing position */
		1070	if (ctx->tokcxmlvl >= TOKMACNEST)
		1071	errsig(ctx->tokcxerr, ERR_MACNEST);
		1072	ctx->tokcxmsav[ctx->tokcxmlvl] = p;
		1073	ctx->tokcxmsvl[ctx->tokcxmlvl] = len;
		1074	ctx->tokcxmlvl++;
		1075
		1076	/* point to the token's expansion and keep going */
		1077	p = df->expan;
		1078	len = df->explen;
		1079	goto nexttoken;
		1080	}
		1081
		1082	/* look up in symbol table(s), if any */
		1083	for (tab = ctx->tokcxstab ; tab ; tab = tab->toktnxt)
		1084	{
		1085	if ((found = (*tab->toktfsea)(tab, tok->toknam, l, hash,
		1086	&tok->toksym)) != 0)
		1087	break;
		1088	}
		1089
		1090	if (found && tok->toksym.tokstyp == TOKSTKW)
		1091	tok->toktyp = tok->toksym.toksval;
		1092	else
		1093	{
		1094	tok->toktyp = TOKTSYMBOL;
		1095	if (!found) tok->toksym.tokstyp = TOKSTUNK;
		1096	}
		1097	goto done;
		1098	}
		1099	else if (isdigit((uchar)*p))
		1100	{
		1101	long acc = 0;
		1102
		1103	/* check for octal/hex */
		1104	if (*p == '0')
		1105	{
		1106	++p, --len;
		1107	if (len && (p == 'x' \|\| p == 'X'))
		1108	{
		1109	/* hex */
		1110	++p, --len;
		1111	while (len && TOKISHEX(*p))
		1112	{
		1113	acc = (acc << 4) + TOKHEX2INT(*p);
		1114	++p, --len;
		1115	}
		1116	}
		1117	else
		1118	{
		1119	/* octal */
		1120	while (len && TOKISOCT(*p))
		1121	{
		1122	acc = (acc << 3) + TOKOCT2INT(*p);
		1123	++p, --len;
		1124	}
		1125	}
		1126	}
		1127	else
		1128	{
		1129	/* decimal */
		1130	while (len && isdigit((uchar)*p))
		1131	{
		1132	acc = (acc << 1) + (acc << 3) + TOKDEC2INT(*p);
		1133	++p, --len;
		1134	}
		1135	}
		1136	tok->tokval = acc;
		1137	tok->toktyp = TOKTNUMBER;
		1138	goto done;
		1139	}
		1140	else if (p == '"' \|\| p == '\'')
		1141	{
		1142	char delim; /* closing delimiter we're looking for */
		1143	char strstart; / pointer to start of string */
		1144	int warned;
		1145
		1146	delim = *p;
		1147	--len;
		1148	strstart = ++p;
		1149
		1150	if (delim == '"' && len >= 2 && p == '<' && (p+1) == '<')
		1151	{
		1152	/* save the current parsing position */
		1153	if (ctx->tokcxmlvl >= TOKMACNEST)
		1154	errsig(ctx->tokcxerr, ERR_MACNEST);
		1155	ctx->tokcxmsav[ctx->tokcxmlvl] = p + 2;
		1156	ctx->tokcxmsvl[ctx->tokcxmlvl] = len - 2;
		1157	ctx->tokcxmlvl++;
		1158
		1159	/*
		1160	* read from the special "<<" expansion string - use the
		1161	* version for a "<<" at the very beginning of the string
		1162	*/
		1163	p = tokmac1s;
		1164	len = strlen(p);
		1165	ctx->tokcxflg \|= TOKCXFINMAC;
		1166	goto nexttoken;
		1167	}
		1168	tok->toktyp = (delim == '"' ? TOKTDSTRING : TOKTSSTRING);
		1169
		1170	tok->tokofs = (ctx->tokcxsst)(ctx->tokcxscx); / start the string */
		1171	for (warned = FALSE ;; )
		1172	{
		1173	if (len >= 2 && *p == '\\')
		1174	{
		1175	if ((p+1) == '"' \|\| (p+1) == '\'')
		1176	{
		1177	(*ctx->tokcxsad)(ctx->tokcxscx, strstart,
		1178	(ushort)(p - strstart));
		1179	strstart = p + 1;
		1180	}
		1181	p += 2;
		1182	len -= 2;
		1183	}
		1184	else if (len == 0 \|\| *p == delim \|\|
		1185	(delim == '"' && len >= 2 && p == '<' && (p+1) == '<'
		1186	&& !(ctx->tokcxflg & TOKCXFINMAC)))
		1187	{
		1188	(*ctx->tokcxsad)(ctx->tokcxscx, strstart,
		1189	(ushort)(p - strstart));
		1190	if (len == 0)
		1191	{
		1192	if (ctx->tokcxmlvl)
		1193	{
		1194	ctx->tokcxmlvl--;
		1195	p = ctx->tokcxmsav[ctx->tokcxmlvl];
		1196	len = ctx->tokcxmsvl[ctx->tokcxmlvl];
		1197	}
		1198	else
		1199	(*ctx->tokcxsad)(ctx->tokcxscx, " ", (ushort)1);
		1200
		1201	while (len == 0)
		1202	{
		1203	if (tokgetlin(ctx, FALSE))
		1204	errsig(ctx->tokcxerr, ERR_STREOF);
		1205	p = ctx->tokcxptr;
		1206	len = ctx->tokcxlen;
		1207
		1208	/* warn if it looks like the end of an object */
		1209	if (!warned && len && (p == ';' \|\| p == '}'))
		1210	{
		1211	errlog(ctx->tokcxerr, ERR_STREND);
		1212	warned = TRUE; /* warn only once per string */
		1213	}
		1214
		1215	/* scan past whitespace at start of line */
		1216	while (len && t_isspace(*p)) ++p, --len;
		1217	}
		1218	strstart = p;
		1219	}
		1220	else break;
		1221	}
		1222	else
		1223	++p, --len;
		1224	}
		1225
		1226	/* end the string */
		1227	(*ctx->tokcxsend)(ctx->tokcxscx);
		1228
		1229	/* check to see how it ended */
		1230	if (len != 0 && *p == delim)
		1231	{
		1232	/*
		1233	* We ended with the matching delimiter. Move past the
		1234	* closing delimiter.
		1235	*/
		1236	++p;
		1237	--len;
		1238
		1239	/*
		1240	* If we have a pending close paren we need to put in
		1241	* because of an embedded expression that occurred earlier
		1242	* in the string, parse the macro to provide the paren.
		1243	*/
		1244	if ((ctx->tokcxflg & TOKCXF_EMBED_PAREN_AFT) != 0
		1245	&& !(ctx->tokcxflg & TOKCXFINMAC))
		1246	{
		1247	/* clear the flag */
		1248	ctx->tokcxflg &= ~TOKCXF_EMBED_PAREN_AFT;
		1249
		1250	/* push the current parsing position */
		1251	if (ctx->tokcxmlvl >= TOKMACNEST)
		1252	errsig(ctx->tokcxerr, ERR_MACNEST);
		1253	ctx->tokcxmsav[ctx->tokcxmlvl] = p;
		1254	ctx->tokcxmsvl[ctx->tokcxmlvl] = len;
		1255	ctx->tokcxmlvl++;
		1256
		1257	/* parse the macro */
		1258	p = tokmac4;
		1259	len = strlen(p);
		1260	}
		1261	}
		1262	else if (len != 0 && *p == '<')
		1263	{
		1264	/* save the current parsing position */
		1265	if (ctx->tokcxmlvl >= TOKMACNEST)
		1266	errsig(ctx->tokcxerr, ERR_MACNEST);
		1267	ctx->tokcxmsav[ctx->tokcxmlvl] = p + 2;
		1268	ctx->tokcxmsvl[ctx->tokcxmlvl] = len - 2;
		1269	ctx->tokcxmlvl++;
		1270
		1271	/* read from the "<<" expansion */
		1272	p = tokmac1;
		1273	len = strlen(p);
		1274	ctx->tokcxflg \|= TOKCXFINMAC;
		1275
		1276	/*
		1277	* Set the special push-a-paren flag: we'll return an open
		1278	* paren now, so that we have an open paren before the
		1279	* string, and then on the next call to toknext() we'll
		1280	* immediately return the string we've already parsed here.
		1281	* This will ensure that everything in the string is
		1282	* properly grouped together as a single indivisible
		1283	* expression.
		1284	*
		1285	* Note that we only need to do this for the first embedded
		1286	* expression in a string. Once we have a close paren
		1287	* pending, we don't need more open parens.
		1288	*/
		1289	if (!(ctx->tokcxflg & TOKCXF_EMBED_PAREN_AFT))
		1290	{
		1291	ctx->tokcxflg \|= TOKCXF_EMBED_PAREN_PRE;
		1292	tok->toktyp = TOKTLPAR;
		1293	}
		1294	}
		1295	goto done;
		1296	}
		1297	else if (len >= 2 && p == '>' && (p+1) == '>'
		1298	&& (ctx->tokcxflg & TOKCXFINMAC) != 0)
		1299	{
		1300	/* skip the ">>" */
		1301	ctx->tokcxflg &= ~TOKCXFINMAC;
		1302	p += 2;
		1303	len -= 2;
		1304
		1305	/* save the current parsing position */
		1306	if (ctx->tokcxmlvl >= TOKMACNEST)
		1307	errsig(ctx->tokcxerr, ERR_MACNEST);
		1308	ctx->tokcxmsav[ctx->tokcxmlvl] = p;
		1309	ctx->tokcxmsvl[ctx->tokcxmlvl] = len;
		1310	ctx->tokcxmlvl++;
		1311
		1312	if (*p == '"')
		1313	{
		1314	++(ctx->tokcxmsav[ctx->tokcxmlvl - 1]);
		1315	--(ctx->tokcxmsvl[ctx->tokcxmlvl - 1]);
		1316	p = tokmac3;
		1317
		1318	/*
		1319	* we won't need an extra closing paren now, since tokmac3
		1320	* provides it
		1321	*/
		1322	ctx->tokcxflg &= ~TOKCXF_EMBED_PAREN_AFT;
		1323	}
		1324	else
		1325	{
		1326	/*
		1327	* The string is continuing. Set a flag to note that we
		1328	* need to provide a close paren after the end of the
		1329	* string, and parse the glue (tokmac2) that goes between
		1330	* the expression and the resumption of the string.
		1331	*/
		1332	ctx->tokcxflg \|= TOKCXF_EMBED_PAREN_AFT;
		1333	p = tokmac2;
		1334	}
		1335
		1336	len = strlen(p);
		1337	goto nexttoken;
		1338	}
		1339	else
		1340	{
		1341	tokscdef *sc;
		1342
		1343	for (sc = ctx->tokcxsc[ctx->tokcxinx[(uchar)*p]] ; sc ;
		1344	sc = sc->tokscnxt)
		1345	{
		1346	if (toksceq(sc->tokscstr, p, sc->toksclen, len))
		1347	{
		1348	tok->toktyp = sc->toksctyp;
		1349	p += sc->toksclen;
		1350	len -= sc->toksclen;
		1351	goto done;
		1352	}
		1353	}
		1354	errsig(ctx->tokcxerr, ERR_INVTOK);
		1355	}
		1356
		1357	done:
		1358	ctx->tokcxptr = p;
		1359	ctx->tokcxlen = len;
		1360	return(tok->toktyp);
		1361	}
		1362
		1363	/* initialize a linear symbol table */
		1364	void toktlini(errcxdef errctx, toktldef toktab, uchar *mem, uint siz)
		1365	{
		1366	CLRSTRUCT(*toktab);
		1367
		1368	/* initialize superclass data */
		1369	toktab->toktlsc.toktfadd = toktladd; /* set add-symbol method */
		1370	toktab->toktlsc.toktfsea = toktlsea; /* set search-table method */
		1371	toktab->toktlsc.toktfeach = toktleach; /* set 'each' method */
		1372	toktab->toktlsc.toktfset = toktlset; /* set 'update' method */
		1373	toktab->toktlsc.tokterr = errctx; /* set error handling context */
		1374
		1375	/* initialize class data */
		1376	toktab->toktlptr = mem;
		1377	toktab->toktlnxt = mem;
		1378	toktab->toktlsiz = siz;
		1379	}
		1380
		1381	/* add a symbol to a linear symbol table */
		1382	void toktladd(toktdef toktab1, char name, int namel,
		1383	int typ, int val, int hash)
		1384	{
		1385	uint siz = sizeof(toks1def) + namel;
		1386	toksdef *sym;
		1387	toktldef toktab = (toktldef )toktab1;
		1388
		1389	VARUSED(hash);
		1390
		1391	if (toktab->toktlsiz < siz)
		1392	errsig(toktab->toktlsc.tokterr, ERR_NOLCLSY);
		1393
		1394	sym = (toksdef *)toktab->toktlnxt;
		1395	siz = osrndsz(siz);
		1396	toktab->toktlnxt += siz;
		1397	if (siz > toktab->toktlsiz) toktab->toktlsiz = 0;
		1398	else toktab->toktlsiz -= siz;
		1399
		1400	/* set up symbol */
		1401	sym->toksval = val;
		1402	sym->tokslen = namel;
		1403	sym->tokstyp = typ;
		1404	sym->toksfr = 0;
		1405	memcpy(sym->toksnam, name, (size_t)(namel + 1));
		1406
		1407	/* indicate there's one more symbol in the table */
		1408	++(toktab->toktlcnt);
		1409	}
		1410
		1411	/* delete all symbols from a linear symbol table */
		1412	void toktldel(toktldef *tab)
		1413	{
		1414	tab->toktlcnt = 0;
		1415	tab->toktlsiz += tab->toktlnxt - tab->toktlptr;
		1416	tab->toktlnxt = tab->toktlptr;
		1417	}
		1418
		1419	/* call a function for every symbol in a linear symbol table */
		1420	void toktleach(toktdef *tab1,
		1421	void (cb)(void ctx, toksdef sym), void ctx)
		1422	{
		1423	toksdef *p;
		1424	uint cnt;
		1425	toktldef tab = (toktldef )tab1;
		1426
		1427	for (p = (toksdef *)tab->toktlptr, cnt = tab->toktlcnt ; cnt ; --cnt )
		1428	{
		1429	(*cb)(ctx, p);
		1430	p = (toksdef )(((uchar )p)
		1431	+ osrndsz(p->tokslen + sizeof(toks1def)));
		1432	}
		1433	}
		1434
		1435	/* search a linear symbol table */
		1436	int toktlsea(toktdef tab1, char name, int namel, int hash, toksdef *ret)
		1437	{
		1438	toksdef *p;
		1439	uint cnt;
		1440	toktldef tab = (toktldef )tab1;
		1441
		1442	VARUSED(hash);
		1443
		1444	for (p = (toksdef *)tab->toktlptr, cnt = tab->toktlcnt ; cnt ; --cnt )
		1445	{
		1446	if (p->tokslen == namel && !memcmp(p->toksnam, name, (size_t)namel))
		1447	{
		1448	memcpy(ret, p, (size_t)(sizeof(toks1def) + namel));
		1449	return(TRUE);
		1450	}
		1451
		1452	p = (toksdef )(((uchar )p)
		1453	+ osrndsz(p->tokslen + sizeof(toks1def)));
		1454	}
		1455
		1456	/* nothing found - indicate by returning FALSE */
		1457	return(FALSE);
		1458	}
		1459
		1460	/* update a symbol in a linear symbol table */
		1461	void toktlset(toktdef tab1, toksdef newsym)
		1462	{
		1463	toksdef *p;
		1464	uint cnt;
		1465	toktldef tab = (toktldef )tab1;
		1466
		1467	for (p = (toksdef *)tab->toktlptr, cnt = tab->toktlcnt ; cnt ; --cnt )
		1468	{
		1469	if (p->tokslen == newsym->tokslen
		1470	&& !memcmp(p->toksnam, newsym->toksnam, (size_t)newsym->tokslen))
		1471	{
		1472	p->toksval = newsym->toksval;
		1473	p->tokstyp = newsym->tokstyp;
		1474	return;
		1475	}
		1476
		1477	p = (toksdef )(((uchar )p)
		1478	+ osrndsz(p->tokslen + sizeof(toks1def)));
		1479	}
		1480	}
		1481
		1482	tokcxdef tokcxini(errcxdef errctx, mcmcxdef mcmctx, tokldef sctab)
		1483	{
		1484	int i;
		1485	int cnt;
		1486	tokldef *p;
		1487	uchar c;
		1488	uchar index[256];
		1489	tokcxdef *ret;
		1490	tokscdef *sc;
		1491	ushort siz;
		1492
		1493	/* set up index table: finds tokcxsc entry from character value */
		1494	memset(index, 0, (size_t)sizeof(index));
		1495	for (i = cnt = 0, p = sctab ; (c = p->toklstr[0]) != 0 ; ++cnt, ++p)
		1496	if (!index[c]) index[c] = ++i;
		1497
		1498	/* allocate memory for table plus the tokscdef's */
		1499	siz = sizeof(tokcxdef) + (i * sizeof(tokscdef *))
		1500	+ ((cnt + 1) * sizeof(tokscdef));
		1501	ret = (tokcxdef *)mchalo(errctx, siz, "tokcxini");
		1502	memset(ret, 0, (size_t)siz);
		1503
		1504	/* copy the index, set up fixed part */
		1505	memcpy(ret->tokcxinx, index, sizeof(ret->tokcxinx));
		1506	ret->tokcxerr = errctx;
		1507	ret->tokcxmem = mcmctx;
		1508
		1509	/* start out without an #if */
		1510	ret->tokcxifcur = TOKIF_IF_YES;
		1511
		1512	/* force the first toknext() to read a line */
		1513	ret->tokcxptr = "\000";
		1514
		1515	/* figure where the tokscdef's go (right after sc pointer array) */
		1516	sc = (tokscdef *)&ret->tokcxsc[i+1];
		1517
		1518	/* set up the individual tokscdef entries, and link into lists */
		1519	for (p = sctab ; (c = p->toklstr[0]) != 0 ; ++p, ++sc)
		1520	{
		1521	size_t len;
		1522
		1523	sc->toksctyp = p->tokltyp;
		1524	len = sc->toksclen = strlen(p->toklstr);
		1525	memcpy(sc->tokscstr, p->toklstr, len);
		1526	sc->tokscnxt = ret->tokcxsc[index[c]];
		1527	ret->tokcxsc[index[c]] = sc;
		1528	}
		1529
		1530	return(ret);
		1531	}
		1532
		1533	/* add an include path to a tokdef */
		1534	void tokaddinc(tokcxdef ctx, char path, int pathlen)
		1535	{
		1536	tokpdef *newpath;
		1537	tokpdef *last;
		1538
		1539	/* find the tail of the include path list, if any */
		1540	for (last = ctx->tokcxinc ; last && last->tokpnxt ;
		1541	last = last->tokpnxt) ;
		1542
		1543	/* allocate storage for and set up a new path structure */
		1544	newpath = (tokpdef *)mchalo(ctx->tokcxerr,
		1545	(ushort)(sizeof(tokpdef) + pathlen - 1),
		1546	"tokaddinc");
		1547	newpath->tokplen = pathlen;
		1548	newpath->tokpnxt = (tokpdef *)0;
		1549	memcpy(newpath->tokpdir, path, (size_t)pathlen);
		1550
		1551	/* link in at end of list (if no list yet, newpath becomes first entry) */
		1552	if (last)
		1553	last->tokpnxt = newpath;
		1554	else
		1555	ctx->tokcxinc = newpath;
		1556	}
		1557

FrobTADS

cfad47cfa3/t2compiler/tads2/tok.c