Skip to main content
  • Home
  • Development
  • Documentation
  • Donate
  • Operational login
  • Browse the archive

swh logo
SoftwareHeritage
Software
Heritage
Archive
Features
  • Search

  • Downloads

  • Save code now

  • Add forge now

  • Help

swh:1:snp:687ac8cdbfab3b78b7f301abee5f451127f135fc
  • Code
  • Branches (1)
  • Releases (0)
    • Branches
    • Releases
    • HEAD
    • refs/heads/master
    No releases to show
  • c6f07c2
  • /
  • lcc
  • /
  • cpp
  • /
  • tokens.c
Raw File Download

To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.

  • content
  • directory
  • revision
  • snapshot
content badge
swh:1:cnt:ebbd186e6c8612419a30affb8ccd469aa0b09dff
directory badge
swh:1:dir:43affbc4ca01657cd68187bc417d7e70f6145d21
revision badge
swh:1:rev:dbe4ddb10315479fc00086f08e25d968b4b43c49
snapshot badge
swh:1:snp:687ac8cdbfab3b78b7f301abee5f451127f135fc

This interface enables to generate software citations, provided that the root directory of browsed objects contains a citation.cff or codemeta.json file.
Select below a type of object currently browsed in order to generate citations for them.

  • content
  • directory
  • revision
  • snapshot
(requires biblatex-software package)
Generating citation ...
(requires biblatex-software package)
Generating citation ...
(requires biblatex-software package)
Generating citation ...
(requires biblatex-software package)
Generating citation ...
Tip revision: dbe4ddb10315479fc00086f08e25d968b4b43c49 authored by Travis Bradshaw on 31 January 2012, 19:41:34 UTC
The Quake III Arena sources as originally released under the GPL license on August 20, 2005.
Tip revision: dbe4ddb
tokens.c
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cpp.h"

static char wbuf[2*OBS];
static char *wbp = wbuf;

/*
 * 1 for tokens that don't need whitespace when they get inserted
 * by macro expansion
 */
static const char wstab[] = {
	0,	/* END */
	0,	/* UNCLASS */
	0,	/* NAME */
	0,	/* NUMBER */
	0,	/* STRING */
	0,	/* CCON */
	1,	/* NL */
	0,	/* WS */
	0,	/* DSHARP */
	0,	/* EQ */
	0,	/* NEQ */
	0,	/* LEQ */
	0,	/* GEQ */
	0,	/* LSH */
	0,	/* RSH */
	0,	/* LAND */
	0,	/* LOR */
	0,	/* PPLUS */
	0,	/* MMINUS */
	0,	/* ARROW */
	1,	/* SBRA */
	1,	/* SKET */
	1,	/* LP */
	1,	/* RP */
	0,	/* DOT */
	0,	/* AND */
	0,	/* STAR */
	0,	/* PLUS */
	0,	/* MINUS */
	0,	/* TILDE */
	0,	/* NOT */
	0,	/* SLASH */
	0,	/* PCT */
	0,	/* LT */
	0,	/* GT */
	0,	/* CIRC */
	0,	/* OR */
	0,	/* QUEST */
	0,	/* COLON */
	0,	/* ASGN */
	1,	/* COMMA */
	0,	/* SHARP */
	1,	/* SEMIC */
	1,	/* CBRA */
	1,	/* CKET */
	0,	/* ASPLUS */
 	0,	/* ASMINUS */
 	0,	/* ASSTAR */
 	0,	/* ASSLASH */
 	0,	/* ASPCT */
 	0,	/* ASCIRC */
 	0,	/* ASLSH */
	0,	/* ASRSH */
 	0,	/* ASOR */
 	0,	/* ASAND */
	0,	/* ELLIPS */
	0,	/* DSHARP1 */
	0,	/* NAME1 */
	0,	/* DEFINED */
	0,	/* UMINUS */
};

void
maketokenrow(int size, Tokenrow *trp)
{
	trp->max = size;
	if (size>0)
		trp->bp = (Token *)domalloc(size*sizeof(Token));
	else
		trp->bp = NULL;
	trp->tp = trp->bp;
	trp->lp = trp->bp;
}

Token *
growtokenrow(Tokenrow *trp)
{
	int ncur = trp->tp - trp->bp;
	int nlast = trp->lp - trp->bp;

	trp->max = 3*trp->max/2 + 1;
	trp->bp = (Token *)realloc(trp->bp, trp->max*sizeof(Token));
	if (trp->bp == NULL)
		error(FATAL, "Out of memory from realloc");
	trp->lp = &trp->bp[nlast];
	trp->tp = &trp->bp[ncur];
	return trp->lp;
}

/*
 * Compare a row of tokens, ignoring the content of WS; return !=0 if different
 */
int
comparetokens(Tokenrow *tr1, Tokenrow *tr2)
{
	Token *tp1, *tp2;

	tp1 = tr1->tp;
	tp2 = tr2->tp;
	if (tr1->lp-tp1 != tr2->lp-tp2)
		return 1;
	for (; tp1<tr1->lp ; tp1++, tp2++) {
		if (tp1->type != tp2->type
		 || (tp1->wslen==0) != (tp2->wslen==0)
		 || tp1->len != tp2->len
		 || strncmp((char*)tp1->t, (char*)tp2->t, tp1->len)!=0)
			return 1;
	}
	return 0;
}

/*
 * replace ntok tokens starting at dtr->tp with the contents of str.
 * tp ends up pointing just beyond the replacement.
 * Canonical whitespace is assured on each side.
 */
void
insertrow(Tokenrow *dtr, int ntok, Tokenrow *str)
{
	int nrtok = rowlen(str);

	dtr->tp += ntok;
	adjustrow(dtr, nrtok-ntok);
	dtr->tp -= ntok;
	movetokenrow(dtr, str);
	makespace(dtr);
	dtr->tp += nrtok;
	makespace(dtr);
}

/*
 * make sure there is WS before trp->tp, if tokens might merge in the output
 */
void
makespace(Tokenrow *trp)
{
	uchar *tt;
	Token *tp = trp->tp;

	if (tp >= trp->lp)
		return;
	if (tp->wslen) {
		if (tp->flag&XPWS
		 && (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])) {
			tp->wslen = 0;
			return;
		}
		tp->t[-1] = ' ';
		return;
	}
	if (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])
		return;
	tt = newstring(tp->t, tp->len, 1);
	*tt++ = ' ';
	tp->t = tt;
	tp->wslen = 1;
	tp->flag |= XPWS;
}

/*
 * Copy an entire tokenrow into another, at tp.
 * It is assumed that there is enough space.
 *  Not strictly conforming.
 */
void
movetokenrow(Tokenrow *dtr, Tokenrow *str)
{
	int nby;

	/* nby = sizeof(Token) * (str->lp - str->bp); */
	nby = (char *)str->lp - (char *)str->bp;
	memmove(dtr->tp, str->bp, nby);
}

/*
 * Move the tokens in a row, starting at tr->tp, rightward by nt tokens;
 * nt may be negative (left move).
 * The row may need to be grown.
 * Non-strictly conforming because of the (char *), but easily fixed
 */
void
adjustrow(Tokenrow *trp, int nt)
{
	int nby, size;

	if (nt==0)
		return;
	size = (trp->lp - trp->bp) + nt;
	while (size > trp->max)
		growtokenrow(trp);
	/* nby = sizeof(Token) * (trp->lp - trp->tp); */
	nby = (char *)trp->lp - (char *)trp->tp;
	if (nby)
		memmove(trp->tp+nt, trp->tp, nby);
	trp->lp += nt;
}

/*
 * Copy a row of tokens into the destination holder, allocating
 * the space for the contents.  Return the destination.
 */
Tokenrow *
copytokenrow(Tokenrow *dtr, Tokenrow *str)
{
	int len = rowlen(str);

	maketokenrow(len, dtr);
	movetokenrow(dtr, str);
	dtr->lp += len;
	return dtr;
}

/*
 * Produce a copy of a row of tokens.  Start at trp->tp.
 * The value strings are copied as well.  The first token
 * has WS available.
 */
Tokenrow *
normtokenrow(Tokenrow *trp)
{
	Token *tp;
	Tokenrow *ntrp = new(Tokenrow);
	int len;

	len = trp->lp - trp->tp;
	if (len<=0)
		len = 1;
	maketokenrow(len, ntrp);
	for (tp=trp->tp; tp < trp->lp; tp++) {
		*ntrp->lp = *tp;
		if (tp->len) {
			ntrp->lp->t = newstring(tp->t, tp->len, 1);
			*ntrp->lp->t++ = ' ';
			if (tp->wslen)
				ntrp->lp->wslen = 1;
		}
		ntrp->lp++;
	}
	if (ntrp->lp > ntrp->bp)
		ntrp->bp->wslen = 0;
	return ntrp;
}

/*
 * Debugging
 */
void
peektokens(Tokenrow *trp, char *str)
{
	Token *tp;

	tp = trp->tp;
	flushout();
	if (str)
		fprintf(stderr, "%s ", str);
	if (tp<trp->bp || tp>trp->lp)
		fprintf(stderr, "(tp offset %d) ", tp-trp->bp);
	for (tp=trp->bp; tp<trp->lp && tp<trp->bp+32; tp++) {
		if (tp->type!=NL) {
			int c = tp->t[tp->len];
			tp->t[tp->len] = 0;
			fprintf(stderr, "%s", tp->t);
			tp->t[tp->len] = c;
		}
		if (tp->type==NAME) {
			fprintf(stderr, tp==trp->tp?"{*":"{");
			prhideset(tp->hideset);
			fprintf(stderr, "} ");
		} else
			fprintf(stderr, tp==trp->tp?"{%x*} ":"{%x} ", tp->type);
	}
	fprintf(stderr, "\n");
	fflush(stderr);
}

void
puttokens(Tokenrow *trp)
{
	Token *tp;
	int len;
	uchar *p;

	if (verbose)
		peektokens(trp, "");
	tp = trp->bp;
	for (; tp<trp->lp; tp++) {
		len = tp->len+tp->wslen;
		p = tp->t-tp->wslen;
		while (tp<trp->lp-1 && p+len == (tp+1)->t - (tp+1)->wslen) {
			tp++;
			len += tp->wslen+tp->len;
		}
		if (len>OBS/2) {		/* handle giant token */
			if (wbp > wbuf)
				write(1, wbuf, wbp-wbuf);
			write(1, (char *)p, len);
			wbp = wbuf;
		} else {	
			memcpy(wbp, p, len);
			wbp += len;
		}
		if (wbp >= &wbuf[OBS]) {
			write(1, wbuf, OBS);
			if (wbp > &wbuf[OBS])
				memcpy(wbuf, wbuf+OBS, wbp - &wbuf[OBS]);
			wbp -= OBS;
		}
	}
	trp->tp = tp;
	if (cursource->fd==0)
		flushout();
}

void
flushout(void)
{
	if (wbp>wbuf) {
		write(1, wbuf, wbp-wbuf);
		wbp = wbuf;
	}
}

/*
 * turn a row into just a newline
 */
void
setempty(Tokenrow *trp)
{
	trp->tp = trp->bp;
	trp->lp = trp->bp+1;
	*trp->bp = nltoken;
}

/*
 * generate a number
 */
char *
outnum(char *p, int n)
{
	if (n>=10)
		p = outnum(p, n/10);
	*p++ = n%10 + '0';
	return p;
}

/*
 * allocate and initialize a new string from s, of length l, at offset o
 * Null terminated.
 */
uchar *
newstring(uchar *s, int l, int o)
{
	uchar *ns = (uchar *)domalloc(l+o+1);

	ns[l+o] = '\0';
	return (uchar*)strncpy((char*)ns+o, (char*)s, l) - o;
}

back to top

Software Heritage — Copyright (C) 2015–2026, The Software Heritage developers. License: GNU AGPLv3+.
The source code of Software Heritage itself is available on our development forge.
The source code files archived by Software Heritage are available under their own copyright and licenses.
Terms of use: Archive access, API— Content policy— Contact— JavaScript license information— Web API