rework handling of tags with literal texts, such as <script> and <style> tags - webdump - HTML to plain-text converter for webpages
HTML git clone git://git.codemadness.org/webdump
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit b078d0218a1f4dcfeb59fdbff2fe356b4f30ddc3
DIR parent 520710cbd57b8e9df4886d60e16976df19a701b9
HTML Author: Hiltjo Posthuma <hiltjo@codemadness.org>
Date: Thu, 2 Jul 2026 19:36:46 +0200
rework handling of tags with literal texts, such as <script> and <style> tags
The data in these tags are ignored, but it was noticed handling of literal
texts in tags such as <title>, <template> and <textarea> should be handled
differently.
This requires a bit of a workaround (pushback buffer) to make the parser parse
it as literal text, _except_ for data entities which are handled normally.
This code is ugly spaghetti. On the plus side its unified reusable spaghetti
(hopefully).
Examples:
<title><![CDATA[this is a title with CDATA text. it should all be visible]]></title>
<title>this is a most stupid example <a href="bla">plop</a><b>bold</b> <blink>and</blink> beautiful</title>
<title>This is a test for data entities in titles. < > &lol; Sometimes it works ™</title>
<title>This is a title tag, it has a comment <!-- This should also be visible --></title>
(These tests will be added to the webdump_tests repo.)
Diffstat:
M webdump.c | 158 +++++++++++++++++++++----------
1 file changed, 110 insertions(+), 48 deletions(-)
---
DIR diff --git a/webdump.c b/webdump.c
@@ -329,7 +329,7 @@ static struct tag tags[] = {
{ "tbody", TagTbody, DisplayInline, 0, DisplayTable, 0, 1, 0, 0, 0 },
{ "td", TagTd, DisplayTableCell, 0, DisplayTableRow, 0, 1, 0, 0, 0 },
{ "template", TagTemplate, DisplayNone, 0, 0, 0, 0, 0, 0, 0 },
-{ "textarea", TagTextarea, DisplayInline, 0, 0, 0, 0, 0, 0, 0 },
+{ "textarea", TagTextarea, DisplayInline | DisplayPre, 0, 0, 0, 0, 0, 0, 0 },
{ "tfoot", TagTfoot, DisplayInline, 0, DisplayTable, 0, 1, 0, 0, 0 },
{ "th", TagTh, DisplayTableCell, MarkupBold, DisplayTableRow, 0, 1, 0, 0, 0 },
{ "thead", TagThead, DisplayInline, 0, DisplayTable, 0, 1, 0, 0, 0 },
@@ -387,34 +387,6 @@ errx(int exitstatus, const char *fmt, ...)
exit(exitstatus);
}
-static const char *ignorestate, *endtag;
-static int (*getnext)(void);
-
-/* return a space for all data until some case-insensitive string occurs. This
- is used to parse incorrect HTML/XML that contains unescaped HTML in script
- or style tags. If you see some </script> tag in a CDATA or comment
- section then e-mail W3C and tell them the web is too complex. */
-static inline int
-getnext_ignore(void)
-{
- int c;
-
- if ((c = getnext()) == EOF)
- return EOF;
-
- if (TOLOWER((unsigned char)c) == TOLOWER((unsigned char)*ignorestate)) {
- ignorestate++;
- if (*ignorestate == '\0') {
- parser.getnext = getnext; /* restore */
- return ' ';
- }
- } else {
- ignorestate = endtag; /* no full match: reset to beginning */
- }
-
- return ' '; /* pretend there is just SPACEs */
-}
-
/* Clear string only; don't free, prevents unnecessary reallocation. */
static void
string_clear(String *s)
@@ -450,6 +422,85 @@ string_append(String *s, const char *data, size_t len)
s->data[s->len] = '\0';
}
+static char *ignorestate;
+static char endtagmatch[16], endtagname[16];
+static unsigned char pushbuf[32], *pushoff, *pushend; /* pushback buffer */
+static unsigned char *pushtagend;
+static int (*getnext)(void);
+
+static void
+getnext_literal_init(void)
+{
+ pushoff = pushbuf;
+ pushend = pushbuf;
+ pushtagend = NULL;
+}
+
+/* Collect data as literal text, used for certain HTML tags such as <script> or <title>.
+ To the parser return a space for all data (so it is parsed as text data)
+ until some case-insensitive string occurs (closing tag).
+ This code is very ugly and I'm not proud of it. Mmmmm spaghetti. */
+static inline int
+getnext_literal(void)
+{
+ int c, len;
+
+ /* there is data in pushback buffer */
+ if (pushend != pushbuf) {
+pushback:
+ if (pushoff >= pushend) {
+ /* reached the offset where the tag ended, fake the callback now */
+ if (pushtagend) {
+ parser.getnext = getnext; /* restore */
+ getnext_literal_init(); /* reset */
+ return '\0';
+ }
+ getnext_literal_init(); /* reset */
+ } else {
+ return (int)*(pushoff++);
+ }
+ }
+
+ if ((c = getnext()) == EOF)
+ return EOF;
+
+ if (TOLOWER((unsigned char)c) == TOLOWER((unsigned char)*ignorestate)) {
+ ignorestate++;
+ if (*ignorestate == '\0') {
+ /* copy complete tag: buffer not checked, it always fits. */
+ memcpy(pushbuf, endtagmatch, strlen(endtagmatch));
+ pushoff = pushbuf;
+ pushend = pushbuf + strlen(endtagmatch) - 1; /* except NUL byte */
+ pushtagend = pushend; /* offset of where the tag ended */
+ }
+ return '\0'; /* NUL byte (this should be ignored by the reader) */
+ } else {
+ /* incomplete tag, fake incomplete leading "<" as data entity "<". */
+ if (ignorestate != endtagmatch) {
+ pushoff = pushbuf;
+ memcpy(pushend, "<", 4); /* buffer not checked, it always fits. */
+ pushend = pushbuf + 4;
+
+ len = ignorestate - endtagmatch - 1;
+ if (len > 0) {
+ /* buffer not checked, it always fits. */
+ memcpy(pushend, endtagmatch + 1, len); /* copy except leading "<" */
+ pushend += len;
+ }
+ ignorestate = endtagmatch; /* no full match: reset to beginning */
+ }
+ }
+
+ if (pushend != pushbuf) {
+ /* append current character to end of pushback buffer */
+ *pushend = c;
+ pushend++;
+ goto pushback;
+ }
+
+ return c;
+}
+
static char *
estrdup(const char *s)
{
@@ -1115,7 +1166,7 @@ printpre(const char *s, size_t len)
cur->hasdata = 1;
}
- for (; *s && i < len; s++, i++) {
+ for (; i < len; s++, i++) {
switch (*s) {
case '\n':
putchar('\n');
@@ -2101,23 +2152,6 @@ xmltagstartparsed(XMLParser *p, const char *t, size_t tl, int isshort)
if ((found = findtag(t)))
tagid = found->id;
- /* temporary replace the callback except the reader and end of tag
- restore the context once we receive the same ignored tag in the
- end tag handler */
- if (tagid == TagScript) {
- ignorestate = endtag = "</script>";
- getnext = p->getnext; /* for restore */
- p->getnext = getnext_ignore;
- xmltagend(p, t, tl, 0); /* fake the call the tag was ended */
- return;
- } else if (tagid == TagStyle) {
- ignorestate = endtag = "</style>";
- getnext = p->getnext; /* for restore */
- p->getnext = getnext_ignore;
- xmltagend(p, t, tl, 0); /* fake the call the tag was ended */
- return;
- }
-
#if 0
/* disable line-wrapping inside tables */
if (tagid == TagTable)
@@ -2293,8 +2327,36 @@ xmltagstartparsed(XMLParser *p, const char *t, size_t tl, int isshort)
}
/* autoclose tags, such as <br>, pretend we are <br/> */
- if (!isshort && cur->tag.isvoid)
+ if (!isshort && cur->tag.isvoid) {
xmltagend(p, t, tl, 1); /* pretend close of short tag */
+ return;
+ }
+
+ /* Temporary replace the callback except the reader and end of tag
+ restore the context once we receive the same ignored tag in the
+ end tag handler.
+
+ HTML script tags often contain characters such as "<".
+ These are interpreted as literal "<" text and not HTML.
+
+ This is why old <style> used to be wrapped as comments: <!-- -->.
+ In XHTML CDATA sections would be used. */
+ switch (tagid) {
+ case TagScript:
+ case TagStyle:
+ case TagTitle:
+ case TagTemplate:
+ case TagTextarea:
+ snprintf(endtagmatch, sizeof(endtagmatch), "</%s>", t);
+ snprintf(endtagname, sizeof(endtagname), "%s", t);
+ ignorestate = endtagmatch;
+ getnext = p->getnext; /* for restore */
+ getnext_literal_init();
+ p->getnext = getnext_literal;
+ break;
+ default:
+ break;
+ }
}
static void