improve literal text handling in <title>, sync from webdump - grabtitle - stupid HTML title grabber
HTML git clone git://git.codemadness.org/grabtitle
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit 14f8fe32c30e3571385a8aa6213e8b286414a1c1
DIR parent 7d8e5a616ebc4ab9b97bb5372ae5f68ae261effd
HTML Author: Hiltjo Posthuma <hiltjo@codemadness.org>
Date: Thu, 2 Jul 2026 20:09:53 +0200
improve literal text handling in <title>, sync from webdump
Diffstat:
M LICENSE | 2 +-
M grabtitle.c | 134 +++++++++++++++++++++++--------
M xml.c | 203 ++++++++++++++++++++++++++-----
M xml.h | 23 ++++++++++++++++++++++-
4 files changed, 299 insertions(+), 63 deletions(-)
---
DIR diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
ISC License
-Copyright (c) 2018-2024 Hiltjo Posthuma <hiltjo@codemadness.org>
+Copyright (c) 2018-2026 Hiltjo Posthuma <hiltjo@codemadness.org>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
DIR diff --git a/grabtitle.c b/grabtitle.c
@@ -1,4 +1,3 @@
-#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
@@ -13,38 +12,99 @@
#define pledge(a,b) 0
#endif
+/* ctype-like macros, but always compatible with ASCII / UTF-8 */
+#define ISCNTRL(c) ((c) < ' ' || (c) == 0x7f)
+#define TOLOWER(c) ((((unsigned)c) - 'A' < 26) ? ((c) | 32) : (c))
+
static XMLParser parser;
-static const char *state, *endtag;
+
+static int ignore = 1;
+
+static char *ignorestate;
+static char endtagmatch[16], endtagname[16];
+static unsigned char pushbuf[32], *pushoff, *pushend; /* pushback buffer */
+static unsigned char *pushtagend;
static int (*getnext)(void);
-/* return a space for all data until some case-insensitive string occurs. This
- is used to parse incorrect HTML/XML that contains unescaped HTML in script
- or style tags. If you see some </script> tag in a CDATA or comment
- section then e-mail W3C and tell them the web is too complex. */
+static void
+getnext_literal_init(void)
+{
+ pushoff = pushbuf;
+ pushend = pushbuf;
+ pushtagend = NULL;
+}
+
+/* Collect data as literal text, used for certain HTML tags such as <script> or <title>.
+ To the parser return a space for all data (so it is parsed as text data)
+ until some case-insensitive string occurs (closing tag).
+ This code is very ugly and I'm not proud of it. Mmmmm spaghetti. */
static inline int
-getnext_ignore(void)
+getnext_literal(void)
{
- int c;
+ int c, len;
+
+ /* there is data in pushback buffer */
+ if (pushend != pushbuf) {
+pushback:
+ if (pushoff >= pushend) {
+ /* reached the offset where the tag ended, fake the callback now */
+ if (pushtagend) {
+ parser.getnext = getnext; /* restore */
+ getnext_literal_init(); /* reset */
+ return '\0';
+ }
+ getnext_literal_init(); /* reset */
+ } else {
+ return (int)*(pushoff++);
+ }
+ }
if ((c = getnext()) == EOF)
return EOF;
- if (tolower(c) == tolower((unsigned char)*state)) {
- state++;
- if (*state == '\0') {
- parser.getnext = getnext; /* restore */
- return c;
+ if (TOLOWER((unsigned char)c) == TOLOWER((unsigned char)*ignorestate)) {
+ ignorestate++;
+ if (*ignorestate == '\0') {
+ /* copy complete tag: buffer not checked, it always fits. */
+ memcpy(pushbuf, endtagmatch, strlen(endtagmatch));
+ pushoff = pushbuf;
+ pushend = pushbuf + strlen(endtagmatch) - 1; /* except NUL byte */
+ pushtagend = pushend; /* offset of where the tag ended */
}
+ return '\0'; /* NUL byte (this should be ignored by the reader) */
} else {
- state = endtag;
+ /* incomplete tag, fake incomplete leading "<" as data entity "<". */
+ if (ignorestate != endtagmatch) {
+ pushoff = pushbuf;
+ memcpy(pushend, "<", 4); /* buffer not checked, it always fits. */
+ pushend = pushbuf + 4;
+
+ len = ignorestate - endtagmatch - 1;
+ if (len > 0) {
+ /* buffer not checked, it always fits. */
+ memcpy(pushend, endtagmatch + 1, len); /* copy except leading "<" */
+ pushend += len;
+ }
+ ignorestate = endtagmatch; /* no full match: reset to beginning */
+ }
+ }
+
+ if (pushend != pushbuf) {
+ /* append current character to end of pushback buffer */
+ *pushend = c;
+ pushend++;
+ goto pushback;
}
- return ' ';
+ return c;
}
static void
xmltagend(XMLParser *p, const char *t, size_t tl, int isshort)
{
+ if (ignore)
+ return;
+
putchar('\n');
exit(0);
}
@@ -55,10 +115,11 @@ xmldata(XMLParser *p, const char *d, size_t dl)
{
size_t i;
- for (i = 0; *d && i < dl; i++, d++) {
- if (iscntrl((unsigned char)*d))
- putchar(' ');
- else
+ if (ignore)
+ return;
+
+ for (i = 0; i < dl; i++, d++) {
+ if (!ISCNTRL((unsigned char)*d))
putchar(*d);
}
}
@@ -76,21 +137,24 @@ xmldataentity(XMLParser *p, const char *d, size_t dl)
}
static void
-xmltagstart(XMLParser *p, const char *t, size_t tl)
+xmltagstartparsed(XMLParser *p, const char *t, size_t tl, int isshort)
{
- if (tl == 6 && !strcasecmp(t, "script")) {
- state = endtag = "</script>";
- getnext = p->getnext; /* for restore */
- p->getnext = getnext_ignore;
- } else if (tl == 5 && !strcasecmp(t, "style")) {
- state = endtag = "</style>";
- getnext = p->getnext; /* for restore */
- p->getnext = getnext_ignore;
- } else if (tl == 5 && !strcasecmp(t, "title")) {
- p->xmltagend = xmltagend;
- p->xmlcdata = p->xmldata = xmldata;
- p->xmldataentity = xmldataentity;
+ if (!strcasecmp(t, "title")) {
+ ignore = 0;
+ } else if (!strcasecmp(t, "script") ||
+ !strcasecmp(t, "style") ||
+ !strcasecmp(t, "temple") ||
+ !strcasecmp(t, "textarea")) {
+ } else {
+ return;
}
+
+ snprintf(endtagmatch, sizeof(endtagmatch), "</%s>", t);
+ snprintf(endtagname, sizeof(endtagname), "%s", t);
+ ignorestate = endtagmatch;
+ getnext = p->getnext; /* for restore */
+ getnext_literal_init();
+ p->getnext = getnext_literal;
}
int
@@ -101,7 +165,11 @@ main(void)
return 2;
}
- parser.xmltagstart = xmltagstart;
+ parser.xmlcdata = xmldata;
+ parser.xmldata = xmldata;
+ parser.xmldataentity = xmldataentity;
+ parser.xmltagend = xmltagend;
+ parser.xmltagstartparsed = xmltagstartparsed;
parser.getnext = getchar;
xml_parse(&parser);
DIR diff --git a/xml.c b/xml.c
@@ -5,13 +5,18 @@
#include "xml.h"
+/* ifdef for HTML mode. To differentiate xml.c and webdump HTML changes */
+#define HTML_MODE
+
#define ISALPHA(c) ((((unsigned)c) | 32) - 'a' < 26)
+#define ISDIGIT(c) (((unsigned)c) - '0' < 10)
#define ISSPACE(c) ((c) == ' ' || ((((unsigned)c) - '\t') < 5))
+#define ISXDIGIT(c) ((((unsigned)c) - '0' < 10) || (((unsigned)c) | 32) - 'a' < 6)
static void
xml_parseattrs(XMLParser *x)
{
- size_t namelen = 0;
+ size_t namelen = 0, valuelen;
int c, endsep, endname = 0, valuestart = 0;
while ((c = GETNEXT()) != EOF) {
@@ -22,33 +27,94 @@ xml_parseattrs(XMLParser *x)
} else if (c == '?')
; /* ignore */
else if (c == '=') {
+ x->name[namelen] = '\0';
valuestart = 1;
endname = 1;
} else if (namelen && ((endname && !valuestart && ISALPHA(c)) || (c == '>' || c == '/'))) {
+ /* attribute without value */
+ x->name[namelen] = '\0';
+ if (x->xmlattrstart)
+ x->xmlattrstart(x, x->tag, x->taglen, x->name, namelen);
+ if (x->xmlattr)
+ x->xmlattr(x, x->tag, x->taglen, x->name, namelen, "", 0);
+ if (x->xmlattrend)
+ x->xmlattrend(x, x->tag, x->taglen, x->name, namelen);
endname = 0;
+ x->name[0] = c;
namelen = 1;
} else if (namelen && valuestart) {
/* attribute with value */
+ if (x->xmlattrstart)
+ x->xmlattrstart(x, x->tag, x->taglen, x->name, namelen);
+
+ valuelen = 0;
if (c == '\'' || c == '"') {
endsep = c;
- while ((c = GETNEXT()) != EOF) {
- if (c == endsep)
- break;
- }
} else {
- while ((c = GETNEXT()) != EOF) {
- if (c == '>' || ISSPACE(c))
- break;
+ endsep = ' '; /* ISSPACE() */
+ goto startvalue;
+ }
+
+ while ((c = GETNEXT()) != EOF) {
+startvalue:
+ if (c == '&') { /* entities */
+ x->data[valuelen] = '\0';
+ /* call data function with data before entity if there is data */
+ if (valuelen && x->xmlattr)
+ x->xmlattr(x, x->tag, x->taglen, x->name, namelen, x->data, valuelen);
+ x->data[0] = c;
+ valuelen = 1;
+ while ((c = GETNEXT()) != EOF) {
+ if (c == endsep || (endsep == ' ' && (c == '>' || ISSPACE(c))))
+ break;
+ if (valuelen < sizeof(x->data) - 1)
+ x->data[valuelen++] = c;
+ else {
+ /* entity too long for buffer, handle as normal data */
+ x->data[valuelen] = '\0';
+ if (x->xmlattr)
+ x->xmlattr(x, x->tag, x->taglen, x->name, namelen, x->data, valuelen);
+ x->data[0] = c;
+ valuelen = 1;
+ break;
+ }
+ if (c == ';') {
+ x->data[valuelen] = '\0';
+ if (x->xmlattrentity)
+ x->xmlattrentity(x, x->tag, x->taglen, x->name, namelen, x->data, valuelen);
+ valuelen = 0;
+ break;
+ }
+ }
+ } else if (c != endsep && !(endsep == ' ' && (c == '>' || ISSPACE(c)))) {
+ if (valuelen < sizeof(x->data) - 1) {
+ x->data[valuelen++] = c;
+ } else {
+ x->data[valuelen] = '\0';
+ if (x->xmlattr)
+ x->xmlattr(x, x->tag, x->taglen, x->name, namelen, x->data, valuelen);
+ x->data[0] = c;
+ valuelen = 1;
+ }
+ }
+ if (c == endsep || (endsep == ' ' && (c == '>' || ISSPACE(c)))) {
+ x->data[valuelen] = '\0';
+ if (x->xmlattr)
+ x->xmlattr(x, x->tag, x->taglen, x->name, namelen, x->data, valuelen);
+ if (x->xmlattrend)
+ x->xmlattrend(x, x->tag, x->taglen, x->name, namelen);
+ break;
}
}
namelen = endname = valuestart = 0;
- } else {
- namelen = 1;
+ } else if (namelen < sizeof(x->name) - 1) {
+ x->name[namelen++] = c;
}
if (c == '>') {
break;
} else if (c == '/') {
x->isshorttag = 1;
+ x->name[0] = '\0';
namelen = 0;
}
}
@@ -57,18 +123,49 @@ xml_parseattrs(XMLParser *x)
static void
xml_parsecomment(XMLParser *x)
{
- size_t i = 0;
+ size_t datalen = 0, i = 0;
int c;
+ if (x->xmlcommentstart)
+ x->xmlcommentstart(x);
while ((c = GETNEXT()) != EOF) {
+ if (c == '-' || c == '>') {
+ if (x->xmlcomment && datalen) {
+ x->data[datalen] = '\0';
+ x->xmlcomment(x, x->data, datalen);
+ datalen = 0;
+ }
+ }
+
if (c == '-') {
- if (i < 2)
- i++;
+ if (++i > 2) {
+ if (x->xmlcomment)
+ for (; i > 2; i--)
+ x->xmlcomment(x, "-", 1);
+ i = 2;
+ }
+ continue;
} else if (c == '>' && i == 2) {
+ if (x->xmlcommentend)
+ x->xmlcommentend(x);
return;
- } else {
+ } else if (i) {
+ if (x->xmlcomment) {
+ for (; i > 0; i--)
+ x->xmlcomment(x, "-", 1);
+ }
i = 0;
}
+
+ if (datalen < sizeof(x->data) - 1) {
+ x->data[datalen++] = c;
+ } else {
+ x->data[datalen] = '\0';
+ if (x->xmlcomment)
+ x->xmlcomment(x, x->data, datalen);
+ x->data[0] = c;
+ datalen = 1;
+ }
}
}
@@ -78,9 +175,11 @@ xml_parsecdata(XMLParser *x)
size_t datalen = 0, i = 0;
int c;
+ if (x->xmlcdatastart)
+ x->xmlcdatastart(x);
while ((c = GETNEXT()) != EOF) {
if (c == ']' || c == '>') {
- if (x->xmlcdata) {
+ if (x->xmlcdata && datalen) {
x->data[datalen] = '\0';
x->xmlcdata(x, x->data, datalen);
datalen = 0;
@@ -96,8 +195,10 @@ xml_parsecdata(XMLParser *x)
}
continue;
} else if (c == '>' && i == 2) {
+ if (x->xmlcdataend)
+ x->xmlcdataend(x);
return;
- } else {
+ } else if (i) {
if (x->xmlcdata)
for (; i > 0; i--)
x->xmlcdata(x, "]", 1);
@@ -151,7 +252,7 @@ struct namedentity {
long cp;
};
-int
+static int
namedentitycmp(const void *v1, const void *v2)
{
struct namedentity *n1 = (struct namedentity *)v1;
@@ -160,12 +261,13 @@ namedentitycmp(const void *v1, const void *v2)
return strcmp(n1->entity, n2->entity);
}
+static const struct namedentity entities[] = {
+#include "namedentities.h"
+};
+
static int
namedentitytostr(const char *e, char *buf, size_t bufsiz)
{
- static const struct namedentity entities[] = {
-#include "namedentities.h"
- };
struct namedentity find, *found;
size_t i;
@@ -188,22 +290,39 @@ static int
numericentitytostr(const char *e, char *buf, size_t bufsiz)
{
long l;
- int len;
+ int base, len;
+ const char *s;
char *end;
/* buffer is too small */
if (bufsiz < 5)
return -1;
+ /* hex (base 16) or decimal (base 10) */
+ if (*e == 'x') {
+ e++;
+ for (s = e; *s && *s != ';'; s++) {
+ if (!ISXDIGIT((unsigned char)*s))
+ return -1; /* invalid: no hex */
+ }
+ base = 16;
+
+ } else {
+ for (s = e; *s && *s != ';'; s++) {
+ if (!ISDIGIT((unsigned char)*s))
+ return -1; /* invalid: no digits */
+ }
+ base = 10;
+ }
+ if (*s != ';' || *(s + 1) != '\0')
+ return -1; /* must end with ';' NUL */
+
errno = 0;
- /* hex (16) or decimal (10) */
- if (*e == 'x')
- l = strtol(++e, &end, 16);
- else
- l = strtol(e, &end, 10);
+ l = strtol(e, &end, base);
+
/* invalid value or not a well-formed entity or invalid code point */
if (errno || e == end || *end != ';' || l < 0 || l > 0x10ffff ||
- (l >= 0xd800 && l <= 0xdfff))
+ (l >= 0xd800 && l <= 0xdfff)) /* surrogate range */
return -1;
len = codepointtoutf8(l, buf);
buf[len] = '\0';
@@ -232,8 +351,13 @@ xml_parse(XMLParser *x)
size_t datalen, tagdatalen;
int c, isend;
+#ifdef HTML_MODE
+ goto read_data;
+#else
+ /* HTML: process data before a tag occured aswell */
while ((c = GETNEXT()) != EOF && c != '<')
; /* skip until < */
+#endif
while (c != EOF) {
if (c == '<') { /* parse tag */
@@ -293,6 +417,8 @@ xml_parse(XMLParser *x)
x->xmltagstart(x, x->tag, x->taglen);
if (ISSPACE(c))
xml_parseattrs(x);
+ if (x->xmltagstartparsed)
+ x->xmltagstartparsed(x, x->tag, x->taglen, x->isshorttag);
}
/* call tagend for short tag or processing instruction */
if (x->isshorttag) {
@@ -307,10 +433,15 @@ xml_parse(XMLParser *x)
}
}
} else {
+#ifdef HTML_MODE
+read_data:
+#endif
/* parse tag data */
datalen = 0;
+ if (x->xmldatastart)
+ x->xmldatastart(x);
while ((c = GETNEXT()) != EOF) {
- if (c == '&') {
+ if (c == '&') { /* entities */
if (datalen) {
x->data[datalen] = '\0';
if (x->xmldata)
@@ -355,9 +486,25 @@ xml_parse(XMLParser *x)
x->data[datalen] = '\0';
if (x->xmldata && datalen)
x->xmldata(x, x->data, datalen);
+ if (x->xmldataend)
+ x->xmldataend(x);
+#ifdef HTML_MODE
+ datalen = 0;
+#endif
break;
}
}
+
+#ifdef HTML_MODE
+ /* pending data, even if a tag didn't close (EOF, etc). */
+ if (datalen) {
+ x->data[datalen] = '\0';
+ if (x->xmldata && datalen)
+ x->xmldata(x, x->data, datalen);
+ if (x->xmldataend)
+ x->xmldataend(x);
+ }
+#endif
}
}
}
DIR diff --git a/xml.h b/xml.h
@@ -5,20 +5,41 @@
typedef struct xmlparser {
/* handlers */
+ void (*xmlattr)(struct xmlparser *, const char *, size_t,
+ const char *, size_t, const char *, size_t);
+ void (*xmlattrend)(struct xmlparser *, const char *, size_t,
+ const char *, size_t);
+ void (*xmlattrstart)(struct xmlparser *, const char *, size_t,
+ const char *, size_t);
+ void (*xmlattrentity)(struct xmlparser *, const char *, size_t,
+ const char *, size_t, const char *, size_t);
+ void (*xmlcdatastart)(struct xmlparser *);
void (*xmlcdata)(struct xmlparser *, const char *, size_t);
+ void (*xmlcdataend)(struct xmlparser *);
+ void (*xmlcommentstart)(struct xmlparser *);
+ void (*xmlcomment)(struct xmlparser *, const char *, size_t);
+ void (*xmlcommentend)(struct xmlparser *);
void (*xmldata)(struct xmlparser *, const char *, size_t);
+ void (*xmldataend)(struct xmlparser *);
void (*xmldataentity)(struct xmlparser *, const char *, size_t);
+ void (*xmldatastart)(struct xmlparser *);
void (*xmltagend)(struct xmlparser *, const char *, size_t, int);
void (*xmltagstart)(struct xmlparser *, const char *, size_t);
+ void (*xmltagstartparsed)(struct xmlparser *, const char *,
+ size_t, int);
-#define GETNEXT (x)->getnext
+#ifndef GETNEXT
+ #define GETNEXT (x)->getnext
int (*getnext)(void);
+#endif
/* current tag */
char tag[1024];
size_t taglen;
/* current tag is a short tag ? <tag /> */
int isshorttag;
+ /* current attribute name */
+ char name[1024];
/* data buffer used for tag data, CDATA and attribute data */
char data[BUFSIZ];
} XMLParser;