Don't dump more than 100 char in g_warnings about missing HTML tags errors
[claws.git] / src / html.c
index 790697a1c83e0764b0db963f130e468716f7a6cc..fd623b1931d301352972651fbc5fd2562dd2a272 100644 (file)
@@ -1,10 +1,10 @@
 /*
- * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
- * Copyright (C) 1999-2005 Hiroyuki Yamamoto
+ * Claws Mail -- a GTK+ based, lightweight, and fast e-mail client
+ * Copyright (C) 1999-2016 Hiroyuki Yamamoto and the Claws Mail team
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
+ * the Free Software Foundation; either version 3 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
@@ -13,8 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <glib.h>
 #include "codeconv.h"
 #include "utils.h"
 
-#define HTMLBUFSIZE    8192
-#define HR_STR         "------------------------------------------------"
+#define SC_HTMLBUFSIZE 8192
+#define HR_STR         "────────────────────────────────────────────────"
+#define LI_STR         "• "
 
-typedef struct _HTMLSymbol     HTMLSymbol;
+typedef struct _SC_HTMLSymbol  SC_HTMLSymbol;
 
-struct _HTMLSymbol
+struct _SC_HTMLSymbol
 {
        gchar *const key;
        gchar *const val;
 };
 
-static HTMLSymbol symbol_list[] = {
-       {"&lt;"    , "<"},
-       {"&gt;"    , ">"},
-       {"&amp;"   , "&"},
-       {"&quot;"  , "\""},
-       {"&nbsp;"  , " "},
-       {"&trade;" , "(TM)"},
-
-       {"&#153;", "(TM)"},
+static SC_HTMLSymbol symbol_list[] = {
+ {"&#34;", "\42"},
+ {"&#38;", "\46"},
+ {"&#39;", "\47"},
+ {"&#60;", "\74"},
+ {"&#62;", "\76"},
+ {"&#034;", "\42"},
+ {"&#038;", "\46"},
+ {"&#039;", "\47"},
+ {"&#060;", "\74"},
+ {"&#062;", "\76"},
+ {"&#146;", "\47"},
+ {"&#153;", "\342\204\242"},
+ {"&#160;", "\40"},
+ {"&#161;", "\302\241"},
+ {"&#162;", "\302\242"},
+ {"&#163;", "\302\243"},
+ {"&#164;", "\302\244"},
+ {"&#165;", "\302\245"},
+ {"&#166;", "\302\246"},
+ {"&#167;", "\302\247"},
+ {"&#168;", "\302\250"},
+ {"&#169;", "\302\251"},
+ {"&#170;", "\302\252"},
+ {"&#171;", "\302\253"},
+ {"&#172;", "\302\254"},
+ {"&#173;", "\302\255"},
+ {"&#174;", "\302\256"},
+ {"&#175;", "\302\257"},
+ {"&#176;", "\302\260"},
+ {"&#177;", "\302\261"},
+ {"&#178;", "\302\262"},
+ {"&#179;", "\302\263"},
+ {"&#180;", "\302\264"},
+ {"&#181;", "\302\265"},
+ {"&#182;", "\302\266"},
+ {"&#183;", "\302\267"},
+ {"&#184;", "\302\270"},
+ {"&#185;", "\302\271"},
+ {"&#186;", "\302\272"},
+ {"&#187;", "\302\273"},
+ {"&#188;", "\302\274"},
+ {"&#189;", "\302\275"},
+ {"&#190;", "\302\276"},
+ {"&#191;", "\302\277"},
+ {"&#192;", "\303\200"},
+ {"&#193;", "\303\201"},
+ {"&#194;", "\303\202"},
+ {"&#195;", "\303\203"},
+ {"&#196;", "\303\204"},
+ {"&#197;", "\303\205"},
+ {"&#198;", "\303\206"},
+ {"&#199;", "\303\207"},
+ {"&#200;", "\303\210"},
+ {"&#201;", "\303\211"},
+ {"&#202;", "\303\212"},
+ {"&#203;", "\303\213"},
+ {"&#204;", "\303\214"},
+ {"&#205;", "\303\215"},
+ {"&#206;", "\303\216"},
+ {"&#207;", "\303\217"},
+ {"&#208;", "\303\220"},
+ {"&#209;", "\303\221"},
+ {"&#210;", "\303\222"},
+ {"&#211;", "\303\223"},
+ {"&#212;", "\303\224"},
+ {"&#213;", "\303\225"},
+ {"&#214;", "\303\226"},
+ {"&#215;", "\303\227"},
+ {"&#216;", "\303\230"},
+ {"&#217;", "\303\231"},
+ {"&#218;", "\303\232"},
+ {"&#219;", "\303\233"},
+ {"&#220;", "\303\234"},
+ {"&#221;", "\303\235"},
+ {"&#222;", "\303\236"},
+ {"&#223;", "\303\237"},
+ {"&#224;", "\303\240"},
+ {"&#225;", "\303\241"},
+ {"&#226;", "\303\242"},
+ {"&#227;", "\303\243"},
+ {"&#228;", "\303\244"},
+ {"&#229;", "\303\245"},
+ {"&#230;", "\303\246"},
+ {"&#231;", "\303\247"},
+ {"&#232;", "\303\250"},
+ {"&#233;", "\303\251"},
+ {"&#234;", "\303\252"},
+ {"&#235;", "\303\253"},
+ {"&#236;", "\303\254"},
+ {"&#237;", "\303\255"},
+ {"&#238;", "\303\256"},
+ {"&#239;", "\303\257"},
+ {"&#240;", "\303\260"},
+ {"&#241;", "\303\261"},
+ {"&#242;", "\303\262"},
+ {"&#243;", "\303\263"},
+ {"&#244;", "\303\264"},
+ {"&#245;", "\303\265"},
+ {"&#246;", "\303\266"},
+ {"&#247;", "\303\267"},
+ {"&#248;", "\303\270"},
+ {"&#249;", "\303\271"},
+ {"&#250;", "\303\272"},
+ {"&#251;", "\303\273"},
+ {"&#252;", "\303\274"},
+ {"&#253;", "\303\275"},
+ {"&#254;", "\303\276"},
+ {"&#255;", "\303\277"},
+ {"&#338;", "\305\222"},
+ {"&#339;", "\305\223"},
+ {"&#352;", "\305\240"},
+ {"&#353;", "\305\241"},
+ {"&#376;", "\305\270"},
+ {"&#710;", "\313\206"},
+ {"&#732;", "\313\234"},
+ {"&#8194;", "\342\200\202"},
+ {"&#8195;", "\342\200\203"},
+ {"&#8201;", "\342\200\211"},
+ {"&#8211;", "\342\200\223"},
+ {"&#8212;", "\342\200\224"},
+ {"&#8216;", "\342\200\230"},
+ {"&#8217;", "\342\200\231"},
+ {"&#8218;", "\342\200\232"},
+ {"&#8220;", "\342\200\234"},
+ {"&#8221;", "\342\200\235"},
+ {"&#8222;", "\342\200\236"},
+ {"&#8224;", "\342\200\240"},
+ {"&#8225;", "\342\200\241"},
+ {"&#8226;", "\342\200\242"},
+ {"&#8230;", "\342\200\246"},
+ {"&#8240;", "\342\200\260"},
+ {"&#8249;", "\342\200\271"},
+ {"&#8250;", "\342\200\272"},
+ {"&#8364;", "\342\202\254"},
+ {"&#8482;", "\342\204\242"},
+ {"&quot;", "\42"},
+ {"&amp;", "\46"},
+ {"&apos;", "\47"},
+ {"&lt;", "\74"},
+ {"&gt;", "\76"},
+ {"&squot;", "\47"},
+ {"&nbsp;", "\40"},
+ {"&iexcl;", "\302\241"},
+ {"&cent;", "\302\242"},
+ {"&pound;", "\302\243"},
+ {"&curren;", "\302\244"},
+ {"&yen;", "\302\245"},
+ {"&brvbar;", "\302\246"},
+ {"&sect;", "\302\247"},
+ {"&uml;", "\302\250"},
+ {"&copy;", "\302\251"},
+ {"&ordf;", "\302\252"},
+ {"&laquo;", "\302\253"},
+ {"&not;", "\302\254"},
+ {"&shy;", "\302\255"},
+ {"&reg;", "\302\256"},
+ {"&macr;", "\302\257"},
+ {"&deg;", "\302\260"},
+ {"&plusmn;", "\302\261"},
+ {"&sup2;", "\302\262"},
+ {"&sup3;", "\302\263"},
+ {"&acute;", "\302\264"},
+ {"&micro;", "\302\265"},
+ {"&para;", "\302\266"},
+ {"&middot;", "\302\267"},
+ {"&cedil;", "\302\270"},
+ {"&sup1;", "\302\271"},
+ {"&ordm;", "\302\272"},
+ {"&raquo;", "\302\273"},
+ {"&frac14;", "\302\274"},
+ {"&frac12;", "\302\275"},
+ {"&frac34;", "\302\276"},
+ {"&iquest;", "\302\277"},
+ {"&Agrave;", "\303\200"},
+ {"&Aacute;", "\303\201"},
+ {"&Acirc;", "\303\202"},
+ {"&Atilde;", "\303\203"},
+ {"&Auml;", "\303\204"},
+ {"&Aring;", "\303\205"},
+ {"&AElig;", "\303\206"},
+ {"&Ccedil;", "\303\207"},
+ {"&Egrave;", "\303\210"},
+ {"&Eacute;", "\303\211"},
+ {"&Ecirc;", "\303\212"},
+ {"&Euml;", "\303\213"},
+ {"&Igrave;", "\303\214"},
+ {"&Iacute;", "\303\215"},
+ {"&Icirc;", "\303\216"},
+ {"&Iuml;", "\303\217"},
+ {"&ETH;", "\303\220"},
+ {"&Ntilde;", "\303\221"},
+ {"&Ograve;", "\303\222"},
+ {"&Oacute;", "\303\223"},
+ {"&Ocirc;", "\303\224"},
+ {"&Otilde;", "\303\225"},
+ {"&Ouml;", "\303\226"},
+ {"&times;", "\303\227"},
+ {"&Oslash;", "\303\230"},
+ {"&Ugrave;", "\303\231"},
+ {"&Uacute;", "\303\232"},
+ {"&Ucirc;", "\303\233"},
+ {"&Uuml;", "\303\234"},
+ {"&Yacute;", "\303\235"},
+ {"&THORN;", "\303\236"},
+ {"&szlig;", "\303\237"},
+ {"&agrave;", "\303\240"},
+ {"&aacute;", "\303\241"},
+ {"&acirc;", "\303\242"},
+ {"&atilde;", "\303\243"},
+ {"&auml;", "\303\244"},
+ {"&aring;", "\303\245"},
+ {"&aelig;", "\303\246"},
+ {"&ccedil;", "\303\247"},
+ {"&egrave;", "\303\250"},
+ {"&eacute;", "\303\251"},
+ {"&ecirc;", "\303\252"},
+ {"&euml;", "\303\253"},
+ {"&igrave;", "\303\254"},
+ {"&iacute;", "\303\255"},
+ {"&icirc;", "\303\256"},
+ {"&iuml;", "\303\257"},
+ {"&eth;", "\303\260"},
+ {"&ntilde;", "\303\261"},
+ {"&ograve;", "\303\262"},
+ {"&oacute;", "\303\263"},
+ {"&ocirc;", "\303\264"},
+ {"&otilde;", "\303\265"},
+ {"&ouml;", "\303\266"},
+ {"&divide;", "\303\267"},
+ {"&oslash;", "\303\270"},
+ {"&ugrave;", "\303\271"},
+ {"&uacute;", "\303\272"},
+ {"&ucirc;", "\303\273"},
+ {"&uuml;", "\303\274"},
+ {"&yacute;", "\303\275"},
+ {"&thorn;", "\303\276"},
+ {"&yuml;", "\303\277"},
+ {"&OElig;", "\305\222"},
+ {"&oelig;", "\305\223"},
+ {"&Scaron;", "\305\240"},
+ {"&scaron;", "\305\241"},
+ {"&Yuml;", "\305\270"},
+ {"&circ;", "\313\206"},
+ {"&tilde;", "\313\234"},
+ {"&ensp;", "\342\200\202"},
+ {"&emsp;", "\342\200\203"},
+ {"&thinsp;", "\342\200\211"},
+ {"&ndash;", "\342\200\223"},
+ {"&mdash;", "\342\200\224"},
+ {"&lsquo;", "\342\200\230"},
+ {"&rsquo;", "\342\200\231"},
+ {"&sbquo;", "\342\200\232"},
+ {"&ldquo;", "\342\200\234"},
+ {"&rdquo;", "\342\200\235"},
+ {"&bdquo;", "\342\200\236"},
+ {"&dagger;", "\342\200\240"},
+ {"&Dagger;", "\342\200\241"},
+ {"&bull;", "\342\200\242"},
+ {"&hellip;", "\342\200\246"},
+ {"&permil;", "\342\200\260"},
+ {"&lsaquo;", "\342\200\271"},
+ {"&rsaquo;", "\342\200\272"},
+ {"&euro;", "\342\202\254"},
+ {"&trade;", "\342\204\242"}
 };
 
-static HTMLSymbol ascii_symbol_list[] = {
-       {"&iexcl;" , "^!"},
-       {"&brvbar;", "|"},
-       {"&copy;"  , "(C)"},
-       {"&laquo;" , "<<"},
-       {"&reg;"   , "(R)"},
-
-       {"&sup2;"  , "^2"},
-       {"&sup3;"  , "^3"},
-       {"&acute;" , "'"},
-       {"&cedil;" , ","},
-       {"&sup1;"  , "^1"},
-       {"&raquo;" , ">>"},
-       {"&frac14;", "1/4"},
-       {"&frac12;", "1/2"},
-       {"&frac34;", "3/4"},
-       {"&iquest;", "^?"},
-
-       {"&Agrave;", "A`"},
-       {"&Aacute;", "A'"},
-       {"&Acirc;" , "A^"},
-       {"&Atilde;", "A~"},
-       {"&AElig;" , "AE"},
-       {"&Egrave;", "E`"},
-       {"&Eacute;", "E'"},
-       {"&Ecirc;" , "E^"},
-       {"&Igrave;", "I`"},
-       {"&Iacute;", "I'"},
-       {"&Icirc;" , "I^"},
-
-       {"&Ntilde;", "N~"},
-       {"&Ograve;", "O`"},
-       {"&Oacute;", "O'"},
-       {"&Ocirc;" , "O^"},
-       {"&Otilde;", "O~"},
-       {"&Ugrave;", "U`"},
-       {"&Uacute;", "U'"},
-       {"&Ucirc;" , "U^"},
-       {"&Yacute;", "Y'"},
-
-       {"&agrave;", "a`"},
-       {"&aacute;", "a'"},
-       {"&acirc;" , "a^"},
-       {"&atilde;", "a~"},
-       {"&aelig;" , "ae"},
-       {"&egrave;", "e`"},
-       {"&eacute;", "e'"},
-       {"&ecirc;" , "e^"},
-       {"&igrave;", "i`"},
-       {"&iacute;", "i'"},
-       {"&icirc;" , "i^"},
-
-       {"&ntilde;", "n~"},
-       {"&ograve;", "o`"},
-       {"&oacute;", "o'"},
-       {"&ocirc;" , "o^"},
-       {"&otilde;", "o~"},
-       {"&ugrave;", "u`"},
-       {"&uacute;", "u'"},
-       {"&ucirc;" , "u^"},
-       {"&yacute;", "y'"},
+typedef struct _SC_HTMLAltSymbol       SC_HTMLAltSymbol;
+
+struct _SC_HTMLAltSymbol
+{
+       gint key;
+       gchar *const val;
 };
 
 static GHashTable *default_symbol_table;
 
-static HTMLState html_read_line                (HTMLParser     *parser);
-static void html_append_char           (HTMLParser     *parser,
+static SC_HTMLState sc_html_read_line  (SC_HTMLParser  *parser);
+static void sc_html_append_char                        (SC_HTMLParser  *parser,
                                         gchar           ch);
-static void html_append_str            (HTMLParser     *parser,
+static void sc_html_append_str                 (SC_HTMLParser  *parser,
                                         const gchar    *str,
                                         gint            len);
-static HTMLState html_parse_tag                (HTMLParser     *parser);
-static void html_parse_special         (HTMLParser     *parser);
-static void html_get_parenthesis       (HTMLParser     *parser,
+static SC_HTMLState sc_html_parse_tag  (SC_HTMLParser  *parser);
+static void sc_html_parse_special              (SC_HTMLParser  *parser);
+static void sc_html_get_parenthesis            (SC_HTMLParser  *parser,
                                         gchar          *buf,
                                         gint            len);
 
 
-HTMLParser *html_parser_new(FILE *fp, CodeConverter *conv)
+SC_HTMLParser *sc_html_parser_new(FILE *fp, CodeConverter *conv)
 {
-       HTMLParser *parser;
+       SC_HTMLParser *parser;
 
-       g_return_val_if_fail(fp != NULL, NULL);
-       g_return_val_if_fail(conv != NULL, NULL);
+       cm_return_val_if_fail(fp != NULL, NULL);
+       cm_return_val_if_fail(conv != NULL, NULL);
 
-       parser = g_new0(HTMLParser, 1);
+       parser = g_new0(SC_HTMLParser, 1);
        parser->fp = fp;
        parser->conv = conv;
        parser->str = g_string_new(NULL);
        parser->buf = g_string_new(NULL);
        parser->bufp = parser->buf->str;
-       parser->state = HTML_NORMAL;
+       parser->state = SC_HTML_NORMAL;
        parser->href = NULL;
        parser->newline = TRUE;
        parser->empty_line = TRUE;
        parser->space = FALSE;
        parser->pre = FALSE;
+       parser->indent = 0;
 
 #define SYMBOL_TABLE_ADD(table, list) \
 { \
@@ -153,22 +355,29 @@ HTMLParser *html_parser_new(FILE *fp, CodeConverter *conv)
        for (i = 0; i < sizeof(list) / sizeof(list[0]); i++) \
                g_hash_table_insert(table, list[i].key, list[i].val); \
 }
+#define SYMBOL_TABLE_REF_ADD(table, list) \
+{ \
+       gint i; \
+ \
+       for (i = 0; i < sizeof(list) / sizeof(list[0]); i++) \
+               g_hash_table_insert(table, &list[i].key, list[i].val); \
+}
 
        if (!default_symbol_table) {
                default_symbol_table =
                        g_hash_table_new(g_str_hash, g_str_equal);
                SYMBOL_TABLE_ADD(default_symbol_table, symbol_list);
-               SYMBOL_TABLE_ADD(default_symbol_table, ascii_symbol_list);
        }
 
 #undef SYMBOL_TABLE_ADD
+#undef SYMBOL_TABLE_REF_ADD
 
        parser->symbol_table = default_symbol_table;
 
        return parser;
 }
 
-void html_parser_destroy(HTMLParser *parser)
+void sc_html_parser_destroy(SC_HTMLParser *parser)
 {
        g_string_free(parser->str, TRUE);
        g_string_free(parser->buf, TRUE);
@@ -176,33 +385,33 @@ void html_parser_destroy(HTMLParser *parser)
        g_free(parser);
 }
 
-gchar *html_parse(HTMLParser *parser)
+gchar *sc_html_parse(SC_HTMLParser *parser)
 {
-       parser->state = HTML_NORMAL;
+       parser->state = SC_HTML_NORMAL;
        g_string_truncate(parser->str, 0);
 
        if (*parser->bufp == '\0') {
                g_string_truncate(parser->buf, 0);
                parser->bufp = parser->buf->str;
-               if (html_read_line(parser) == HTML_EOF)
+               if (sc_html_read_line(parser) == SC_HTML_EOF)
                        return NULL;
        }
 
        while (*parser->bufp != '\0') {
                switch (*parser->bufp) {
                case '<': {
-                       HTMLState st;
-                       st = html_parse_tag(parser);
+                       SC_HTMLState st;
+                       st = sc_html_parse_tag(parser);
                        /* when we see an href, we need to flush the str
                         * buffer.  Then collect all the chars until we
                         * see the end anchor tag
                         */
-                       if (HTML_HREF_BEG == st || HTML_HREF == st)
+                       if (SC_HTML_HREF_BEG == st || SC_HTML_HREF == st)
                                return parser->str->str;
                        } 
                        break;
                case '&':
-                       html_parse_special(parser);
+                       sc_html_parse_special(parser);
                        break;
                case ' ':
                case '\t':
@@ -220,23 +429,29 @@ gchar *html_parse(HTMLParser *parser)
                        }
                        /* fallthrough */
                default:
-                       html_append_char(parser, *parser->bufp++);
+                       sc_html_append_char(parser, *parser->bufp++);
                }
        }
 
        return parser->str->str;
 }
 
-static HTMLState html_read_line(HTMLParser *parser)
+static SC_HTMLState sc_html_read_line(SC_HTMLParser *parser)
 {
-       gchar buf[HTMLBUFSIZE];
-       gchar buf2[HTMLBUFSIZE];
+       gchar buf[SC_HTMLBUFSIZE];
+       gchar buf2[SC_HTMLBUFSIZE];
        gint index;
+       gint n;
 
-       if (fgets(buf, sizeof(buf), parser->fp) == NULL) {
-               parser->state = HTML_EOF;
-               return HTML_EOF;
-       }
+       if (parser->fp == NULL)
+               return SC_HTML_EOF;
+
+       n = fread(buf, 1, sizeof(buf) - 1, parser->fp);
+       if (n == 0) {
+               parser->state = SC_HTML_EOF;
+               return SC_HTML_EOF;
+       } else
+               buf[n] = '\0';
 
        if (conv_convert(parser->conv, buf2, sizeof(buf2), buf) < 0) {
                index = parser->bufp - parser->buf->str;
@@ -246,7 +461,7 @@ static HTMLState html_read_line(HTMLParser *parser)
 
                parser->bufp = parser->buf->str + index;
 
-               return HTML_CONV_FAILED;
+               return SC_HTML_CONV_FAILED;
        }
 
        index = parser->bufp - parser->buf->str;
@@ -255,10 +470,10 @@ static HTMLState html_read_line(HTMLParser *parser)
 
        parser->bufp = parser->buf->str + index;
 
-       return HTML_NORMAL;
+       return SC_HTML_NORMAL;
 }
 
-static void html_append_char(HTMLParser *parser, gchar ch)
+static void sc_html_append_char(SC_HTMLParser *parser, gchar ch)
 {
        GString *str = parser->str;
 
@@ -274,11 +489,17 @@ static void html_append_char(HTMLParser *parser, gchar ch)
                parser->newline = TRUE;
                if (str->len > 1 && str->str[str->len - 2] == '\n')
                        parser->empty_line = TRUE;
+               if (parser->indent > 0) {
+                       gint i, n = parser->indent;
+                       for (i = 0; i < n; i++)
+                               g_string_append_c(str, '>');
+                       g_string_append_c(str, ' ');
+               }
        } else
                parser->newline = FALSE;
 }
 
-static void html_append_str(HTMLParser *parser, const gchar *str, gint len)
+static void sc_html_append_str(SC_HTMLParser *parser, const gchar *str, gint len)
 {
        GString *string = parser->str;
 
@@ -305,35 +526,33 @@ static void html_append_str(HTMLParser *parser, const gchar *str, gint len)
                parser->newline = FALSE;
 }
 
-static HTMLTag *html_get_tag(const gchar *str)
+static SC_HTMLTag *sc_html_get_tag(const gchar *str)
 {
-       HTMLTag *tag;
+       SC_HTMLTag *tag;
        gchar *tmp;
        guchar *tmpp;
 
-       g_return_val_if_fail(str != NULL, NULL);
+       cm_return_val_if_fail(str != NULL, NULL);
 
        if (*str == '\0' || *str == '!') return NULL;
 
        Xstrdup_a(tmp, str, return NULL);
 
-       tag = g_new0(HTMLTag, 1);
+       tag = g_new0(SC_HTMLTag, 1);
 
        for (tmpp = tmp; *tmpp != '\0' && !g_ascii_isspace(*tmpp); tmpp++)
                ;
 
        if (*tmpp == '\0') {
-               g_strdown(tmp);
-               tag->name = g_strdup(tmp);
+               tag->name = g_utf8_strdown(tmp, -1);
                return tag;
        } else {
                *tmpp++ = '\0';
-               g_strdown(tmp);
-               tag->name = g_strdup(tmp);
+               tag->name = g_utf8_strdown(tmp, -1);
        }
 
        while (*tmpp != '\0') {
-               HTMLAttr *attr;
+               SC_HTMLAttr *attr;
                gchar *attr_name;
                gchar *attr_value;
                gchar *p;
@@ -360,7 +579,15 @@ static HTMLTag *html_get_tag(const gchar *str)
                                tmpp++;
                                attr_value = tmpp;
                                if ((p = strchr(attr_value, quote)) == NULL) {
-                                       g_warning("html_get_tag(): syntax error in tag: '%s'\n", str);
+                                       if (debug_get_mode()) {
+                                               g_warning("sc_html_get_tag(): syntax error in tag: '%s'",
+                                                                 str);
+                                       } else {
+                                               gchar *cut = g_strndup(str, 100);
+                                               g_warning("sc_html_get_tag(): syntax error in tag: '%s%s'",
+                                                                 cut, strlen(str)>100?"...":".");
+                                               g_free(cut);
+                                       }
                                        return tag;
                                }
                                tmpp = p;
@@ -377,9 +604,8 @@ static HTMLTag *html_get_tag(const gchar *str)
                        attr_value = "";
 
                g_strchomp(attr_name);
-               g_strdown(attr_name);
-               attr = g_new(HTMLAttr, 1);
-               attr->name = g_strdup(attr_name);
+               attr = g_new(SC_HTMLAttr, 1);
+               attr->name = g_utf8_strdown(attr_name, -1);
                attr->value = g_strdup(attr_value);
                tag->attr = g_list_append(tag->attr, attr);
        }
@@ -387,13 +613,13 @@ static HTMLTag *html_get_tag(const gchar *str)
        return tag;
 }
 
-static void html_free_tag(HTMLTag *tag)
+static void sc_html_free_tag(SC_HTMLTag *tag)
 {
        if (!tag) return;
 
        g_free(tag->name);
        while (tag->attr != NULL) {
-               HTMLAttr *attr = (HTMLAttr *)tag->attr->data;
+               SC_HTMLAttr *attr = (SC_HTMLAttr *)tag->attr->data;
                g_free(attr->name);
                g_free(attr->value);
                g_free(attr);
@@ -402,107 +628,141 @@ static void html_free_tag(HTMLTag *tag)
        g_free(tag);
 }
 
-static HTMLState html_parse_tag(HTMLParser *parser)
+static void decode_href(SC_HTMLParser *parser)
 {
-       gchar buf[HTMLBUFSIZE];
-       HTMLTag *tag;
+       gchar *tmp;
+       SC_HTMLParser *tparser = g_new0(SC_HTMLParser, 1);
+
+       tparser->str = g_string_new(NULL);
+       tparser->buf = g_string_new(parser->href);
+       tparser->bufp = tparser->buf->str;
+       tparser->symbol_table = default_symbol_table;
+       
+       tmp = sc_html_parse(tparser);
+       
+       g_free(parser->href);
+       parser->href = g_strdup(tmp);
 
-       html_get_parenthesis(parser, buf, sizeof(buf));
+       sc_html_parser_destroy(tparser);
+}
 
-       tag = html_get_tag(buf);
+static SC_HTMLState sc_html_parse_tag(SC_HTMLParser *parser)
+{
+       gchar buf[SC_HTMLBUFSIZE];
+       SC_HTMLTag *tag;
 
-       parser->state = HTML_UNKNOWN;
-       if (!tag) return HTML_UNKNOWN;
+       sc_html_get_parenthesis(parser, buf, sizeof(buf));
 
-       if (!strcmp(tag->name, "br")) {
+       tag = sc_html_get_tag(buf);
+
+       parser->state = SC_HTML_UNKNOWN;
+       if (!tag) return SC_HTML_UNKNOWN;
+
+       if (!strcmp(tag->name, "br") || !strcmp(tag->name, "br/")) {
                parser->space = FALSE;
-               html_append_char(parser, '\n');
-               parser->state = HTML_BR;
+               sc_html_append_char(parser, '\n');
+               parser->state = SC_HTML_BR;
        } else if (!strcmp(tag->name, "a")) {
                GList *cur;
+               parser->href = NULL;
                for (cur = tag->attr; cur != NULL; cur = cur->next) {
-                       if (cur->data && !strcmp(((HTMLAttr *)cur->data)->name, "href")) {
+                       if (cur->data && !strcmp(((SC_HTMLAttr *)cur->data)->name, "href")) {
                                g_free(parser->href);
-                               parser->href = g_strdup(((HTMLAttr *)cur->data)->value);
-                               parser->state = HTML_HREF_BEG;
+                               parser->href = g_strdup(((SC_HTMLAttr *)cur->data)->value);
+                               decode_href(parser);
+                               parser->state = SC_HTML_HREF_BEG;
                                break;
                        }
                }
+               if (parser->href == NULL)
+                       parser->href = g_strdup("");
+               parser->state = SC_HTML_HREF_BEG;
        } else if (!strcmp(tag->name, "/a")) {
-               parser->state = HTML_HREF;
+               parser->state = SC_HTML_HREF;
        } else if (!strcmp(tag->name, "p")) {
                parser->space = FALSE;
                if (!parser->empty_line) {
                        parser->space = FALSE;
-                       if (!parser->newline) html_append_char(parser, '\n');
-                       html_append_char(parser, '\n');
+                       if (!parser->newline) sc_html_append_char(parser, '\n');
+                       sc_html_append_char(parser, '\n');
                }
-               parser->state = HTML_PAR;
+               parser->state = SC_HTML_PAR;
        } else if (!strcmp(tag->name, "pre")) {
                parser->pre = TRUE;
-               parser->state = HTML_PRE;
+               parser->state = SC_HTML_PRE;
        } else if (!strcmp(tag->name, "/pre")) {
                parser->pre = FALSE;
-               parser->state = HTML_NORMAL;
+               parser->state = SC_HTML_NORMAL;
        } else if (!strcmp(tag->name, "hr")) {
                if (!parser->newline) {
                        parser->space = FALSE;
-                       html_append_char(parser, '\n');
+                       sc_html_append_char(parser, '\n');
                }
-               html_append_str(parser, HR_STR "\n", -1);
-               parser->state = HTML_HR;
+               sc_html_append_str(parser, HR_STR, -1);
+               sc_html_append_char(parser, '\n');
+               parser->state = SC_HTML_HR;
        } else if (!strcmp(tag->name, "div")    ||
                   !strcmp(tag->name, "ul")     ||
                   !strcmp(tag->name, "li")     ||
                   !strcmp(tag->name, "table")  ||
+                  !strcmp(tag->name, "dd")     ||
                   !strcmp(tag->name, "tr")     ||
                   (tag->name[0] == 'h' && g_ascii_isdigit(tag->name[1]))) {
                if (!parser->newline) {
                        parser->space = FALSE;
-                       html_append_char(parser, '\n');
+                       sc_html_append_char(parser, '\n');
+               }
+               if (!strcmp(tag->name, "li")) {
+                       sc_html_append_str(parser, LI_STR, -1);
                }
-               parser->state = HTML_NORMAL;
+               parser->state = SC_HTML_NORMAL;
+       } else if (!strcmp(tag->name, "blockquote")) {
+               parser->state = SC_HTML_NORMAL;
+               parser->indent++;
+       } else if (!strcmp(tag->name, "/blockquote")) {
+               parser->state = SC_HTML_NORMAL;
+               parser->indent--;
        } else if (!strcmp(tag->name, "/table") ||
                   (tag->name[0] == '/' &&
                    tag->name[1] == 'h' &&
                    g_ascii_isdigit(tag->name[1]))) {
                if (!parser->empty_line) {
                        parser->space = FALSE;
-                       if (!parser->newline) html_append_char(parser, '\n');
-                       html_append_char(parser, '\n');
+                       if (!parser->newline) sc_html_append_char(parser, '\n');
+                       sc_html_append_char(parser, '\n');
                }
-               parser->state = HTML_NORMAL;
+               parser->state = SC_HTML_NORMAL;
        } else if (!strcmp(tag->name, "/div")   ||
                   !strcmp(tag->name, "/ul")    ||
                   !strcmp(tag->name, "/li")) {
                if (!parser->newline) {
                        parser->space = FALSE;
-                       html_append_char(parser, '\n');
+                       sc_html_append_char(parser, '\n');
                }
-               parser->state = HTML_NORMAL;
+               parser->state = SC_HTML_NORMAL;
                        }
 
-       html_free_tag(tag);
+       sc_html_free_tag(tag);
 
        return parser->state;
 }
 
-static void html_parse_special(HTMLParser *parser)
+static void sc_html_parse_special(SC_HTMLParser *parser)
 {
        gchar symbol_name[9];
        gint n;
        const gchar *val;
 
-       parser->state = HTML_UNKNOWN;
-       g_return_if_fail(*parser->bufp == '&');
+       parser->state = SC_HTML_UNKNOWN;
+       cm_return_if_fail(*parser->bufp == '&');
 
        /* &foo; */
        for (n = 0; parser->bufp[n] != '\0' && parser->bufp[n] != ';'; n++)
                ;
        if (n > 7 || parser->bufp[n] != ';') {
                /* output literal `&' */
-               html_append_char(parser, *parser->bufp++);
-               parser->state = HTML_NORMAL;
+               sc_html_append_char(parser, *parser->bufp++);
+               parser->state = SC_HTML_NORMAL;
                return;
        }
        strncpy2(symbol_name, parser->bufp, n + 2);
@@ -510,141 +770,63 @@ static void html_parse_special(HTMLParser *parser)
 
        if ((val = g_hash_table_lookup(parser->symbol_table, symbol_name))
            != NULL) {
-               html_append_str(parser, val, -1);
-               parser->state = HTML_NORMAL;
+               sc_html_append_str(parser, val, -1);
+               parser->state = SC_HTML_NORMAL;
                return;
-       } else if (symbol_name[1] == '#' && g_ascii_isdigit(symbol_name[2])) {
-               gint ch;
-
-               ch = atoi(symbol_name + 2);
-               if ((ch > 0 && ch <= 127) ||
-                   (ch >= 128 && ch <= 255 &&
-                    parser->conv->charset == C_ISO_8859_1)) {
-                       html_append_char(parser, ch);
-                       parser->state = HTML_NORMAL;
-                       return;
-               } else {
-                       char *symb = NULL;
-                       switch (ch) {
-                       /* http://www.w3schools.com/html/html_entitiesref.asp */
-                       case 338:       /* capital ligature OE  &OElig;  */
-                               symb = "OE";  
-                               break;
-                       case 339:       /* small ligature OE    &oelig;  */
-                               symb = "oe";  
-                               break;
-                       case 352:       /* capital S w/caron    &Scaron; */
-                       case 353:       /* small S w/caron      &scaron; */
-                       case 376:       /* cap Y w/ diaeres     &Yuml;   */
-                               break;
-                       case 710:       /* circumflex accent    &circ;   */
-                               symb = "^";  
-                               break;
-                       case 732:       /* small tilde          &tilde;  */
-                               symb = "~";  
-                               break;
-                       case 8194:      /* en space             &ensp;   */
-                       case 8195:      /* em space             &emsp;   */
-                       case 8201:      /* thin space           &thinsp; */
-                               symb = " ";  
-                               break;
-                       case 8204:      /* zero width non-joiner &zwnj;  */
-                       case 8205:      /* zero width joiner    &zwj;   */
-                       case 8206:      /* l-t-r mark           &lrm;   */
-                       case 8207:      /* r-t-l mark           &rlm     */
-                               break;
-                       case 8211:      /* en dash              &ndash;  */
-                               symb = "-";  
-                               break;
-                       case 8212:      /* em dash              &mdash;  */
-                               symb = "--";  
-                               break;
-                       case 8216:      /* l single quot mark   &lsquo;  */
-                               symb = "`";  
-                               break;
-                       case 8217:      /* r single quot mark   &rsquo;  */
-                               symb = "'";  
-                               break;
-                       case 8218:      /* single low-9 quot    &sbquo;  */
-                               symb = ",";  
-                               break;
-                       case 8220:      /* l double quot mark   &ldquo;  */
-                               symb = "``";  
-                               break;
-                       case 8221:      /* r double quot mark   &rdquo;  */
-                               symb = "''";  
-                               break;
-                       case 8222:      /* double low-9 quot    &bdquo;  */
-                               symb = ",,";  
-                               break;
-                       case 8224:      /* dagger               &dagger; */
-                       case 8225:      /* double dagger        &Dagger; */
-                               break;
-                       case 8230:      /* horizontal ellipsis  &hellip; */
-                               symb = "...";  
-                               break;
-                       case 8240:      /* per mile             &permil; */
-                               symb = "\%o";  
-                               break;
-                       case 8249:      /* l-pointing angle quot &lsaquo; */
-                               symb = "<";  
-                               break;
-                       case 8250:      /* r-pointing angle quot &rsaquo; */
-                               symb = ">";  
-                               break;
-                       case 8364:      /* euro                 &euro;   */
-                               symb = "EUR";  
-                               break;
-                       case 8482:      /* trademark            &trade;  */
-                               symb  = "(TM)";  
-                               break;
-                       default: 
-                               break;
-                       }
-                       if (symb) {
-                               html_append_str(parser, symb, -1);
-                               parser->state = HTML_NORMAL;
-                               return;
-                       }
-               }
-       }
+       } 
 
-       html_append_str(parser, symbol_name, -1);
+       sc_html_append_str(parser, symbol_name, -1);
+}
+
+static gchar *sc_html_find_tag(SC_HTMLParser *parser, const gchar *tag)
+{
+       gchar *cur = parser->bufp;
+       gint len = strlen(tag);
+
+       if (cur == NULL)
+               return NULL;
+
+       while ((cur = strstr(cur, "<")) != NULL) {
+               if (!g_ascii_strncasecmp(cur, tag, len))
+                       return cur;
+               cur += 2;
+       }
+       return NULL;
 }
 
-static void html_get_parenthesis(HTMLParser *parser, gchar *buf, gint len)
+static void sc_html_get_parenthesis(SC_HTMLParser *parser, gchar *buf, gint len)
 {
        gchar *p;
 
        buf[0] = '\0';
-       g_return_if_fail(*parser->bufp == '<');
+       cm_return_if_fail(*parser->bufp == '<');
 
        /* ignore comment / CSS / script stuff */
        if (!strncmp(parser->bufp, "<!--", 4)) {
                parser->bufp += 4;
                while ((p = strstr(parser->bufp, "-->")) == NULL)
-                       if (html_read_line(parser) == HTML_EOF) return;
+                       if (sc_html_read_line(parser) == SC_HTML_EOF) return;
                parser->bufp = p + 3;
                return;
        }
        if (!g_ascii_strncasecmp(parser->bufp, "<style", 6)) {
                parser->bufp += 6;
-               while ((p = strcasestr(parser->bufp, "</style>")) == NULL)
-                       if (html_read_line(parser) == HTML_EOF) return;
+               while ((p = sc_html_find_tag(parser, "</style>")) == NULL)
+                       if (sc_html_read_line(parser) == SC_HTML_EOF) return;
                parser->bufp = p + 8;
                return;
        }
        if (!g_ascii_strncasecmp(parser->bufp, "<script", 7)) {
                parser->bufp += 7;
-               while ((p = strcasestr(parser->bufp, "</script>")) == NULL)
-                       if (html_read_line(parser) == HTML_EOF) return;
+               while ((p = sc_html_find_tag(parser, "</script>")) == NULL)
+                       if (sc_html_read_line(parser) == SC_HTML_EOF) return;
                parser->bufp = p + 9;
                return;
        }
 
        parser->bufp++;
        while ((p = strchr(parser->bufp, '>')) == NULL)
-               if (html_read_line(parser) == HTML_EOF) return;
+               if (sc_html_read_line(parser) == SC_HTML_EOF) return;
 
        strncpy2(buf, parser->bufp, MIN(p - parser->bufp + 1, len));
        g_strstrip(buf);