2006-02-23 [wwp] 2.0.0cvs74
[claws.git] / src / html.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2006 Hiroyuki Yamamoto and the Sylpheed-Claws team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18  */
19
20 #include <glib.h>
21 #include <stdio.h>
22 #include <string.h>
23 #include <ctype.h>
24
25 #include "html.h"
26 #include "codeconv.h"
27 #include "utils.h"
28
29 #define SC_HTMLBUFSIZE  8192
30 #define HR_STR          "------------------------------------------------"
31
32 typedef struct _SC_HTMLSymbol   SC_HTMLSymbol;
33
34 struct _SC_HTMLSymbol
35 {
36         gchar *const key;
37         gchar *const val;
38 };
39
40 static SC_HTMLSymbol symbol_list[] = {
41         {"&lt;"    , "<"},
42         {"&gt;"    , ">"},
43         {"&amp;"   , "&"},
44         {"&quot;"  , "\""},
45         {"&lsquo;",  "'"},
46         {"&rsquo;",  "'"},
47         {"&ldquo;",  "\""},
48         {"&rdquo;",  "\""},
49         {"&nbsp;"  , " "},
50         {"&trade;" , "(TM)"},
51         {"&#153;", "(TM)"},
52         {"&hellip;", "..."},
53 };
54
55 static SC_HTMLSymbol ascii_symbol_list[] = {
56         {"&iexcl;" , "^!"},
57         {"&brvbar;", "|"},
58         {"&copy;"  , "(C)"},
59         {"&laquo;" , "<<"},
60         {"&reg;"   , "(R)"},
61
62         {"&sup2;"  , "^2"},
63         {"&sup3;"  , "^3"},
64         {"&acute;" , "'"},
65         {"&cedil;" , ","},
66         {"&sup1;"  , "^1"},
67         {"&raquo;" , ">>"},
68         {"&frac14;", "1/4"},
69         {"&frac12;", "1/2"},
70         {"&frac34;", "3/4"},
71         {"&iquest;", "^?"},
72
73         {"&Agrave;", "A`"},
74         {"&Aacute;", "A'"},
75         {"&Acirc;" , "A^"},
76         {"&Atilde;", "A~"},
77         {"&AElig;" , "AE"},
78         {"&Egrave;", "E`"},
79         {"&Eacute;", "E'"},
80         {"&Ecirc;" , "E^"},
81         {"&Igrave;", "I`"},
82         {"&Iacute;", "I'"},
83         {"&Icirc;" , "I^"},
84
85         {"&Ntilde;", "N~"},
86         {"&Ograve;", "O`"},
87         {"&Oacute;", "O'"},
88         {"&Ocirc;" , "O^"},
89         {"&Otilde;", "O~"},
90         {"&Ugrave;", "U`"},
91         {"&Uacute;", "U'"},
92         {"&Ucirc;" , "U^"},
93         {"&Yacute;", "Y'"},
94
95         {"&agrave;", "a`"},
96         {"&aacute;", "a'"},
97         {"&acirc;" , "a^"},
98         {"&atilde;", "a~"},
99         {"&aelig;" , "ae"},
100         {"&egrave;", "e`"},
101         {"&eacute;", "e'"},
102         {"&ecirc;" , "e^"},
103         {"&igrave;", "i`"},
104         {"&iacute;", "i'"},
105         {"&icirc;" , "i^"},
106
107         {"&ntilde;", "n~"},
108         {"&ograve;", "o`"},
109         {"&oacute;", "o'"},
110         {"&ocirc;" , "o^"},
111         {"&otilde;", "o~"},
112         {"&ugrave;", "u`"},
113         {"&uacute;", "u'"},
114         {"&ucirc;" , "u^"},
115         {"&yacute;", "y'"},
116 };
117
118 static GHashTable *default_symbol_table;
119
120 static SC_HTMLState sc_html_read_line   (SC_HTMLParser  *parser);
121 static void sc_html_append_char                 (SC_HTMLParser  *parser,
122                                          gchar           ch);
123 static void sc_html_append_str                  (SC_HTMLParser  *parser,
124                                          const gchar    *str,
125                                          gint            len);
126 static SC_HTMLState sc_html_parse_tag   (SC_HTMLParser  *parser);
127 static void sc_html_parse_special               (SC_HTMLParser  *parser);
128 static void sc_html_get_parenthesis             (SC_HTMLParser  *parser,
129                                          gchar          *buf,
130                                          gint            len);
131
132
133 SC_HTMLParser *sc_html_parser_new(FILE *fp, CodeConverter *conv)
134 {
135         SC_HTMLParser *parser;
136
137         g_return_val_if_fail(fp != NULL, NULL);
138         g_return_val_if_fail(conv != NULL, NULL);
139
140         parser = g_new0(SC_HTMLParser, 1);
141         parser->fp = fp;
142         parser->conv = conv;
143         parser->str = g_string_new(NULL);
144         parser->buf = g_string_new(NULL);
145         parser->bufp = parser->buf->str;
146         parser->state = SC_HTML_NORMAL;
147         parser->href = NULL;
148         parser->newline = TRUE;
149         parser->empty_line = TRUE;
150         parser->space = FALSE;
151         parser->pre = FALSE;
152
153 #define SYMBOL_TABLE_ADD(table, list) \
154 { \
155         gint i; \
156  \
157         for (i = 0; i < sizeof(list) / sizeof(list[0]); i++) \
158                 g_hash_table_insert(table, list[i].key, list[i].val); \
159 }
160
161         if (!default_symbol_table) {
162                 default_symbol_table =
163                         g_hash_table_new(g_str_hash, g_str_equal);
164                 SYMBOL_TABLE_ADD(default_symbol_table, symbol_list);
165                 SYMBOL_TABLE_ADD(default_symbol_table, ascii_symbol_list);
166         }
167
168 #undef SYMBOL_TABLE_ADD
169
170         parser->symbol_table = default_symbol_table;
171
172         return parser;
173 }
174
175 void sc_html_parser_destroy(SC_HTMLParser *parser)
176 {
177         g_string_free(parser->str, TRUE);
178         g_string_free(parser->buf, TRUE);
179         g_free(parser->href);
180         g_free(parser);
181 }
182
183 gchar *sc_html_parse(SC_HTMLParser *parser)
184 {
185         parser->state = SC_HTML_NORMAL;
186         g_string_truncate(parser->str, 0);
187
188         if (*parser->bufp == '\0') {
189                 g_string_truncate(parser->buf, 0);
190                 parser->bufp = parser->buf->str;
191                 if (sc_html_read_line(parser) == SC_HTML_EOF)
192                         return NULL;
193         }
194
195         while (*parser->bufp != '\0') {
196                 switch (*parser->bufp) {
197                 case '<': {
198                         SC_HTMLState st;
199                         st = sc_html_parse_tag(parser);
200                         /* when we see an href, we need to flush the str
201                          * buffer.  Then collect all the chars until we
202                          * see the end anchor tag
203                          */
204                         if (SC_HTML_HREF_BEG == st || SC_HTML_HREF == st)
205                                 return parser->str->str;
206                         } 
207                         break;
208                 case '&':
209                         sc_html_parse_special(parser);
210                         break;
211                 case ' ':
212                 case '\t':
213                 case '\r':
214                 case '\n':
215                         if (parser->bufp[0] == '\r' && parser->bufp[1] == '\n')
216                                 parser->bufp++;
217
218                         if (!parser->pre) {
219                                 if (!parser->newline)
220                                         parser->space = TRUE;
221
222                                 parser->bufp++;
223                                 break;
224                         }
225                         /* fallthrough */
226                 default:
227                         sc_html_append_char(parser, *parser->bufp++);
228                 }
229         }
230
231         return parser->str->str;
232 }
233
234 static SC_HTMLState sc_html_read_line(SC_HTMLParser *parser)
235 {
236         gchar buf[SC_HTMLBUFSIZE];
237         gchar buf2[SC_HTMLBUFSIZE];
238         gint index;
239
240         if (fgets(buf, sizeof(buf), parser->fp) == NULL) {
241                 parser->state = SC_HTML_EOF;
242                 return SC_HTML_EOF;
243         }
244
245         if (conv_convert(parser->conv, buf2, sizeof(buf2), buf) < 0) {
246                 index = parser->bufp - parser->buf->str;
247
248                 conv_utf8todisp(buf2, sizeof(buf2), buf);
249                 g_string_append(parser->buf, buf2);
250
251                 parser->bufp = parser->buf->str + index;
252
253                 return SC_HTML_CONV_FAILED;
254         }
255
256         index = parser->bufp - parser->buf->str;
257
258         g_string_append(parser->buf, buf2);
259
260         parser->bufp = parser->buf->str + index;
261
262         return SC_HTML_NORMAL;
263 }
264
265 static void sc_html_append_char(SC_HTMLParser *parser, gchar ch)
266 {
267         GString *str = parser->str;
268
269         if (!parser->pre && parser->space) {
270                 g_string_append_c(str, ' ');
271                 parser->space = FALSE;
272         }
273
274         g_string_append_c(str, ch);
275
276         parser->empty_line = FALSE;
277         if (ch == '\n') {
278                 parser->newline = TRUE;
279                 if (str->len > 1 && str->str[str->len - 2] == '\n')
280                         parser->empty_line = TRUE;
281         } else
282                 parser->newline = FALSE;
283 }
284
285 static void sc_html_append_str(SC_HTMLParser *parser, const gchar *str, gint len)
286 {
287         GString *string = parser->str;
288
289         if (!parser->pre && parser->space) {
290                 g_string_append_c(string, ' ');
291                 parser->space = FALSE;
292         }
293
294         if (len == 0) return;
295         if (len < 0)
296                 g_string_append(string, str);
297         else {
298                 gchar *s;
299                 Xstrndup_a(s, str, len, return);
300                 g_string_append(string, s);
301         }
302
303         parser->empty_line = FALSE;
304         if (string->len > 0 && string->str[string->len - 1] == '\n') {
305                 parser->newline = TRUE;
306                 if (string->len > 1 && string->str[string->len - 2] == '\n')
307                         parser->empty_line = TRUE;
308         } else
309                 parser->newline = FALSE;
310 }
311
312 static SC_HTMLTag *sc_html_get_tag(const gchar *str)
313 {
314         SC_HTMLTag *tag;
315         gchar *tmp;
316         guchar *tmpp;
317
318         g_return_val_if_fail(str != NULL, NULL);
319
320         if (*str == '\0' || *str == '!') return NULL;
321
322         Xstrdup_a(tmp, str, return NULL);
323
324         tag = g_new0(SC_HTMLTag, 1);
325
326         for (tmpp = tmp; *tmpp != '\0' && !g_ascii_isspace(*tmpp); tmpp++)
327                 ;
328
329         if (*tmpp == '\0') {
330                 g_strdown(tmp);
331                 tag->name = g_strdup(tmp);
332                 return tag;
333         } else {
334                 *tmpp++ = '\0';
335                 g_strdown(tmp);
336                 tag->name = g_strdup(tmp);
337         }
338
339         while (*tmpp != '\0') {
340                 SC_HTMLAttr *attr;
341                 gchar *attr_name;
342                 gchar *attr_value;
343                 gchar *p;
344                 gchar quote;
345
346                 while (g_ascii_isspace(*tmpp)) tmpp++;
347                 attr_name = tmpp;
348
349                 while (*tmpp != '\0' && !g_ascii_isspace(*tmpp) &&
350                        *tmpp != '=')
351                         tmpp++;
352                 if (*tmpp != '\0' && *tmpp != '=') {
353                         *tmpp++ = '\0';
354                         while (g_ascii_isspace(*tmpp)) tmpp++;
355                 }
356
357                 if (*tmpp == '=') {
358                         *tmpp++ = '\0';
359                         while (g_ascii_isspace(*tmpp)) tmpp++;
360
361                         if (*tmpp == '"' || *tmpp == '\'') {
362                                 /* name="value" */
363                                 quote = *tmpp;
364                                 tmpp++;
365                                 attr_value = tmpp;
366                                 if ((p = strchr(attr_value, quote)) == NULL) {
367                                         g_warning("sc_html_get_tag(): syntax error in tag: '%s'\n", str);
368                                         return tag;
369                                 }
370                                 tmpp = p;
371                                 *tmpp++ = '\0';
372                                 while (g_ascii_isspace(*tmpp)) tmpp++;
373                         } else {
374                                 /* name=value */
375                                 attr_value = tmpp;
376                                 while (*tmpp != '\0' && !g_ascii_isspace(*tmpp)) tmpp++;
377                                 if (*tmpp != '\0')
378                                         *tmpp++ = '\0';
379                         }
380                 } else
381                         attr_value = "";
382
383                 g_strchomp(attr_name);
384                 g_strdown(attr_name);
385                 attr = g_new(SC_HTMLAttr, 1);
386                 attr->name = g_strdup(attr_name);
387                 attr->value = g_strdup(attr_value);
388                 tag->attr = g_list_append(tag->attr, attr);
389         }
390
391         return tag;
392 }
393
394 static void sc_html_free_tag(SC_HTMLTag *tag)
395 {
396         if (!tag) return;
397
398         g_free(tag->name);
399         while (tag->attr != NULL) {
400                 SC_HTMLAttr *attr = (SC_HTMLAttr *)tag->attr->data;
401                 g_free(attr->name);
402                 g_free(attr->value);
403                 g_free(attr);
404                 tag->attr = g_list_remove(tag->attr, tag->attr->data);
405         }
406         g_free(tag);
407 }
408
409 static SC_HTMLState sc_html_parse_tag(SC_HTMLParser *parser)
410 {
411         gchar buf[SC_HTMLBUFSIZE];
412         SC_HTMLTag *tag;
413
414         sc_html_get_parenthesis(parser, buf, sizeof(buf));
415
416         tag = sc_html_get_tag(buf);
417
418         parser->state = SC_HTML_UNKNOWN;
419         if (!tag) return SC_HTML_UNKNOWN;
420
421         if (!strcmp(tag->name, "br")) {
422                 parser->space = FALSE;
423                 sc_html_append_char(parser, '\n');
424                 parser->state = SC_HTML_BR;
425         } else if (!strcmp(tag->name, "a")) {
426                 GList *cur;
427                 for (cur = tag->attr; cur != NULL; cur = cur->next) {
428                         if (cur->data && !strcmp(((SC_HTMLAttr *)cur->data)->name, "href")) {
429                                 g_free(parser->href);
430                                 parser->href = g_strdup(((SC_HTMLAttr *)cur->data)->value);
431                                 parser->state = SC_HTML_HREF_BEG;
432                                 break;
433                         }
434                 }
435         } else if (!strcmp(tag->name, "/a")) {
436                 parser->state = SC_HTML_HREF;
437         } else if (!strcmp(tag->name, "p")) {
438                 parser->space = FALSE;
439                 if (!parser->empty_line) {
440                         parser->space = FALSE;
441                         if (!parser->newline) sc_html_append_char(parser, '\n');
442                         sc_html_append_char(parser, '\n');
443                 }
444                 parser->state = SC_HTML_PAR;
445         } else if (!strcmp(tag->name, "pre")) {
446                 parser->pre = TRUE;
447                 parser->state = SC_HTML_PRE;
448         } else if (!strcmp(tag->name, "/pre")) {
449                 parser->pre = FALSE;
450                 parser->state = SC_HTML_NORMAL;
451         } else if (!strcmp(tag->name, "hr")) {
452                 if (!parser->newline) {
453                         parser->space = FALSE;
454                         sc_html_append_char(parser, '\n');
455                 }
456                 sc_html_append_str(parser, HR_STR "\n", -1);
457                 parser->state = SC_HTML_HR;
458         } else if (!strcmp(tag->name, "div")    ||
459                    !strcmp(tag->name, "ul")     ||
460                    !strcmp(tag->name, "li")     ||
461                    !strcmp(tag->name, "table")  ||
462                    !strcmp(tag->name, "tr")     ||
463                    (tag->name[0] == 'h' && g_ascii_isdigit(tag->name[1]))) {
464                 if (!parser->newline) {
465                         parser->space = FALSE;
466                         sc_html_append_char(parser, '\n');
467                 }
468                 parser->state = SC_HTML_NORMAL;
469         } else if (!strcmp(tag->name, "/table") ||
470                    (tag->name[0] == '/' &&
471                     tag->name[1] == 'h' &&
472                     g_ascii_isdigit(tag->name[1]))) {
473                 if (!parser->empty_line) {
474                         parser->space = FALSE;
475                         if (!parser->newline) sc_html_append_char(parser, '\n');
476                         sc_html_append_char(parser, '\n');
477                 }
478                 parser->state = SC_HTML_NORMAL;
479         } else if (!strcmp(tag->name, "/div")   ||
480                    !strcmp(tag->name, "/ul")    ||
481                    !strcmp(tag->name, "/li")) {
482                 if (!parser->newline) {
483                         parser->space = FALSE;
484                         sc_html_append_char(parser, '\n');
485                 }
486                 parser->state = SC_HTML_NORMAL;
487                         }
488
489         sc_html_free_tag(tag);
490
491         return parser->state;
492 }
493
494 static void sc_html_parse_special(SC_HTMLParser *parser)
495 {
496         gchar symbol_name[9];
497         gint n;
498         const gchar *val;
499
500         parser->state = SC_HTML_UNKNOWN;
501         g_return_if_fail(*parser->bufp == '&');
502
503         /* &foo; */
504         for (n = 0; parser->bufp[n] != '\0' && parser->bufp[n] != ';'; n++)
505                 ;
506         if (n > 7 || parser->bufp[n] != ';') {
507                 /* output literal `&' */
508                 sc_html_append_char(parser, *parser->bufp++);
509                 parser->state = SC_HTML_NORMAL;
510                 return;
511         }
512         strncpy2(symbol_name, parser->bufp, n + 2);
513         parser->bufp += n + 1;
514
515         if ((val = g_hash_table_lookup(parser->symbol_table, symbol_name))
516             != NULL) {
517                 sc_html_append_str(parser, val, -1);
518                 parser->state = SC_HTML_NORMAL;
519                 return;
520         } else if (symbol_name[1] == '#' && g_ascii_isdigit(symbol_name[2])) {
521                 gint ch;
522
523                 ch = atoi(symbol_name + 2);
524                 if ((ch > 0 && ch <= 127) ||
525                     (ch >= 128 && ch <= 255 &&
526                      parser->conv->charset == C_ISO_8859_1)) {
527                         sc_html_append_char(parser, ch);
528                         parser->state = SC_HTML_NORMAL;
529                         return;
530                 } else {
531                         char *symb = NULL;
532                         switch (ch) {
533                         /* http://www.w3schools.com/html/html_entitiesref.asp */
534                         case 96:        /* backtick  */
535                                 symb = "`";
536                                 break;
537                         case 338:       /* capital ligature OE  &OElig;  */
538                                 symb = "OE";  
539                                 break;
540                         case 339:       /* small ligature OE    &oelig;  */
541                                 symb = "oe";  
542                                 break;
543                         case 352:       /* capital S w/caron    &Scaron; */
544                         case 353:       /* small S w/caron      &scaron; */
545                         case 376:       /* cap Y w/ diaeres     &Yuml;   */
546                                 break;
547                         case 710:       /* circumflex accent    &circ;   */
548                                 symb = "^";  
549                                 break;
550                         case 732:       /* small tilde          &tilde;  */
551                                 symb = "~";  
552                                 break;
553                         case 8194:      /* en space             &ensp;   */
554                         case 8195:      /* em space             &emsp;   */
555                         case 8201:      /* thin space           &thinsp; */
556                                 symb = " ";  
557                                 break;
558                         case 8204:      /* zero width non-joiner &zwnj;  */
559                         case 8205:      /* zero width joiner    &zwj;   */
560                         case 8206:      /* l-t-r mark           &lrm;   */
561                         case 8207:      /* r-t-l mark           &rlm     */
562                                 break;
563                         case 8211:      /* en dash              &ndash;  */
564                                 symb = "-";  
565                                 break;
566                         case 8212:      /* em dash              &mdash;  */
567                                 symb = "--";  
568                                 break;
569                         case 8216:      /* l single quot mark   &lsquo;  */
570                         case 8217:      /* r single quot mark   &rsquo;  */
571                                 symb = "'";  
572                                 break;
573                         case 8218:      /* single low-9 quot    &sbquo;  */
574                                 symb = ",";  
575                                 break;
576                         case 8220:      /* l double quot mark   &ldquo;  */
577                         case 8221:      /* r double quot mark   &rdquo;  */
578                                 symb = "\"";  
579                                 break;
580                         case 8222:      /* double low-9 quot    &bdquo;  */
581                                 symb = ",,";  
582                                 break;
583                         case 8224:      /* dagger               &dagger; */
584                         case 8225:      /* double dagger        &Dagger; */
585                                 break;
586                         case 8230:      /* horizontal ellipsis  &hellip; */
587                                 symb = "...";  
588                                 break;
589                         case 8240:      /* per mile             &permil; */
590                                 symb = "\%o";  
591                                 break;
592                         case 8249:      /* l-pointing angle quot &lsaquo; */
593                                 symb = "<";  
594                                 break;
595                         case 8250:      /* r-pointing angle quot &rsaquo; */
596                                 symb = ">";  
597                                 break;
598                         case 8364:      /* euro                 &euro;   */
599                                 symb = "EUR";  
600                                 break;
601                         case 8482:      /* trademark            &trade;  */
602                                 symb  = "(TM)";  
603                                 break;
604                         default: 
605                                 break;
606                         }
607                         if (symb) {
608                                 sc_html_append_str(parser, symb, -1);
609                                 parser->state = SC_HTML_NORMAL;
610                                 return;
611                         }
612                 }
613         }
614
615         sc_html_append_str(parser, symbol_name, -1);
616 }
617
618 static void sc_html_get_parenthesis(SC_HTMLParser *parser, gchar *buf, gint len)
619 {
620         gchar *p;
621
622         buf[0] = '\0';
623         g_return_if_fail(*parser->bufp == '<');
624
625         /* ignore comment / CSS / script stuff */
626         if (!strncmp(parser->bufp, "<!--", 4)) {
627                 parser->bufp += 4;
628                 while ((p = strstr(parser->bufp, "-->")) == NULL)
629                         if (sc_html_read_line(parser) == SC_HTML_EOF) return;
630                 parser->bufp = p + 3;
631                 return;
632         }
633         if (!g_ascii_strncasecmp(parser->bufp, "<style", 6)) {
634                 parser->bufp += 6;
635                 while ((p = strcasestr(parser->bufp, "</style>")) == NULL)
636                         if (sc_html_read_line(parser) == SC_HTML_EOF) return;
637                 parser->bufp = p + 8;
638                 return;
639         }
640         if (!g_ascii_strncasecmp(parser->bufp, "<script", 7)) {
641                 parser->bufp += 7;
642                 while ((p = strcasestr(parser->bufp, "</script>")) == NULL)
643                         if (sc_html_read_line(parser) == SC_HTML_EOF) return;
644                 parser->bufp = p + 9;
645                 return;
646         }
647
648         parser->bufp++;
649         while ((p = strchr(parser->bufp, '>')) == NULL)
650                 if (sc_html_read_line(parser) == SC_HTML_EOF) return;
651
652         strncpy2(buf, parser->bufp, MIN(p - parser->bufp + 1, len));
653         g_strstrip(buf);
654         parser->bufp = p + 1;
655 }