fix msg display when utf8_instead_of_locale_for_broken_mail is turned off and you...
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2012 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  * 
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #include "claws-features.h"
23 #endif
24
25 #include "defs.h"
26
27 #define _GNU_SOURCE
28 #include <stdio.h>
29
30 #include <glib.h>
31 #include <glib/gi18n.h>
32 #ifdef _WIN32
33 # include <w32lib.h>
34 # define MAP_FAILED     ((char *) -1)
35 #else
36 # include <sys/mman.h>
37 #endif
38 #include <sys/types.h>
39 #include <sys/stat.h>
40
41 #include <time.h>
42
43 #include "msgcache.h"
44 #include "utils.h"
45 #include "procmsg.h"
46 #include "codeconv.h"
47 #include "timing.h"
48 #include "tags.h"
49 #include "prefs_common.h"
50
51 #ifdef HAVE_FWRITE_UNLOCKED
52 #define SC_FWRITE fwrite_unlocked
53 #else
54 #define SC_FWRITE fwrite
55 #endif
56
57 #if G_BYTE_ORDER == G_BIG_ENDIAN
58 #define bswap_32(x) \
59      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
60       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
61      
62 #define MMAP_TO_GUINT32(x)      \
63         (((x[3]&0xff)) |        \
64          ((x[2]&0xff) << 8) |   \
65          ((x[1]&0xff) << 16) |  \
66          ((x[0]&0xff) << 24))
67
68 #define MMAP_TO_GUINT32_SWAPPED(x)      \
69         (((x[0]&0xff)) |                \
70          ((x[1]&0xff) << 8) |           \
71          ((x[2]&0xff) << 16) |          \
72          ((x[3]&0xff) << 24))
73
74 static gboolean msgcache_use_mmap_read = TRUE;
75
76 #else
77 #define bswap_32(x) (x)
78
79 #define MMAP_TO_GUINT32(x)      \
80         (((x[0]&0xff)) |        \
81          ((x[1]&0xff) << 8) |   \
82          ((x[2]&0xff) << 16) |  \
83          ((x[3]&0xff) << 24))
84
85 #define MMAP_TO_GUINT32_SWAPPED(x)      \
86         (((x[0]&0xff)) |                \
87          ((x[1]&0xff) << 8) |           \
88          ((x[2]&0xff) << 16) |          \
89          ((x[3]&0xff) << 24))
90
91 static gboolean msgcache_use_mmap_read = TRUE;
92 #endif
93
94 static gboolean swapping = TRUE;
95
96 typedef enum
97 {
98         DATA_READ,
99         DATA_WRITE,
100         DATA_APPEND
101 } DataOpenMode;
102
103 struct _MsgCache {
104         GHashTable      *msgnum_table;
105         GHashTable      *msgid_table;
106         guint            memusage;
107         time_t           last_access;
108 };
109
110 typedef struct _StringConverter StringConverter;
111 struct _StringConverter {
112         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
113         void   (*free)    (StringConverter *converter);
114 };
115
116 typedef struct _StrdupConverter StrdupConverter;
117 struct _StrdupConverter {
118         StringConverter converter;
119 };
120
121 typedef struct _CharsetConverter CharsetConverter;
122 struct _CharsetConverter {
123         StringConverter converter;
124
125         gchar *srccharset;
126         gchar *dstcharset;
127 };
128
129 MsgCache *msgcache_new(void)
130 {
131         MsgCache *cache;
132         
133         cache = g_new0(MsgCache, 1),
134         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
135         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
136         cache->last_access = time(NULL);
137
138         return cache;
139 }
140
141 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
142 {
143         procmsg_msginfo_free((MsgInfo *)msginfo);
144         return TRUE;
145 }                                                                                         
146
147 void msgcache_destroy(MsgCache *cache)
148 {
149         cm_return_if_fail(cache != NULL);
150
151         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
152         g_hash_table_destroy(cache->msgid_table);
153         g_hash_table_destroy(cache->msgnum_table);
154         g_free(cache);
155 }
156
157 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
158 {
159         MsgInfo *newmsginfo;
160
161         cm_return_if_fail(cache != NULL);
162         cm_return_if_fail(msginfo != NULL);
163
164         newmsginfo = procmsg_msginfo_new_ref(msginfo);
165         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
166         if(newmsginfo->msgid != NULL)
167                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
168         cache->memusage += procmsg_msginfo_memusage(msginfo);
169         cache->last_access = time(NULL);
170
171         msginfo->folder->cache_dirty = TRUE;
172
173         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
174 }
175
176 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
177 {
178         MsgInfo *msginfo;
179
180         cm_return_if_fail(cache != NULL);
181
182         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
183         if(!msginfo)
184                 return;
185
186         cache->memusage -= procmsg_msginfo_memusage(msginfo);
187         if(msginfo->msgid)
188                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
189         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
190         procmsg_msginfo_free(msginfo);
191         cache->last_access = time(NULL);
192
193         msginfo->folder->cache_dirty = TRUE;
194
195         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
196 }
197
198 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
199 {
200         MsgInfo *oldmsginfo, *newmsginfo;
201         
202         cm_return_if_fail(cache != NULL);
203         cm_return_if_fail(msginfo != NULL);
204
205         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
206         if(oldmsginfo && oldmsginfo->msgid) 
207                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
208         if (oldmsginfo) {
209                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
210                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
211                 procmsg_msginfo_free(oldmsginfo);
212         }
213
214         newmsginfo = procmsg_msginfo_new_ref(msginfo);
215         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
216         if(newmsginfo->msgid)
217                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
218         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
219         cache->last_access = time(NULL);
220         
221         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
222
223         msginfo->folder->cache_dirty = TRUE;
224
225         return;
226 }
227
228 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
229 {
230         MsgInfo *msginfo;
231
232         cm_return_val_if_fail(cache != NULL, NULL);
233
234         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
235         if(!msginfo)
236                 return NULL;
237         cache->last_access = time(NULL);
238         
239         return procmsg_msginfo_new_ref(msginfo);
240 }
241
242 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
243 {
244         MsgInfo *msginfo;
245         
246         cm_return_val_if_fail(cache != NULL, NULL);
247         cm_return_val_if_fail(msgid != NULL, NULL);
248
249         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
250         if(!msginfo)
251                 return NULL;
252         cache->last_access = time(NULL);
253         
254         return procmsg_msginfo_new_ref(msginfo);        
255 }
256
257 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
258 {
259         MsgInfoList **listptr = user_data;
260         MsgInfo *msginfo = value;
261
262         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
263 }
264
265 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
266 {
267         MsgInfoList *msg_list = NULL;
268         START_TIMING("");
269         cm_return_val_if_fail(cache != NULL, NULL);
270
271         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
272         cache->last_access = time(NULL);
273         
274         msg_list = g_slist_reverse(msg_list);
275         END_TIMING();
276         return msg_list;
277 }
278
279 time_t msgcache_get_last_access_time(MsgCache *cache)
280 {
281         cm_return_val_if_fail(cache != NULL, 0);
282         
283         return cache->last_access;
284 }
285
286 gint msgcache_get_memory_usage(MsgCache *cache)
287 {
288         cm_return_val_if_fail(cache != NULL, 0);
289
290         return cache->memusage;
291 }
292
293 /*
294  *  Cache saving functions
295  */
296
297 #define READ_CACHE_DATA(data, fp, total_len) \
298 { \
299         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
300                 procmsg_msginfo_free(msginfo); \
301                 error = TRUE; \
302                 goto bail_err; \
303         } \
304         total_len += tmp_len; \
305 }
306
307 #define READ_CACHE_DATA_INT(n, fp) \
308 { \
309         guint32 idata; \
310         size_t ni; \
311  \
312         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
313                 g_warning("read_int: Cache data corrupted, read %zd of %zd at " \
314                           "offset %ld\n", ni, sizeof(idata), ftell(fp)); \
315                 procmsg_msginfo_free(msginfo); \
316                 error = TRUE; \
317                 goto bail_err; \
318         } else \
319                 n = swapping ? bswap_32(idata) : (idata);\
320 }
321
322 #define GET_CACHE_DATA_INT(n)                                                                   \
323 {                                                                                               \
324         if (rem_len < 4) {                                                                      \
325                 g_print("error at rem_len:%d\n", rem_len);                                      \
326                 error = TRUE;                                                                   \
327                 goto bail_err;                                                                  \
328         }                                                                                       \
329         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data)));     \
330         walk_data += 4; rem_len -= 4;                                                           \
331 }
332
333 #define GET_CACHE_DATA(data, total_len) \
334 { \
335         GET_CACHE_DATA_INT(tmp_len);    \
336         if (rem_len < tmp_len) {                                                                \
337                 g_print("error at rem_len:%d (tmp_len %d)\n", rem_len, tmp_len);                \
338                 error = TRUE;                                                                   \
339                 goto bail_err;                                                                  \
340         }                                                                                       \
341         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
342                 g_print("error at rem_len:%d\n", rem_len);\
343                 procmsg_msginfo_free(msginfo); \
344                 error = TRUE; \
345                 goto bail_err; \
346         } \
347         total_len += tmp_len; \
348         walk_data += tmp_len; rem_len -= tmp_len; \
349 }
350
351
352 #define WRITE_CACHE_DATA_INT(n, fp)                     \
353 {                                                       \
354         guint32 idata;                                  \
355                                                         \
356         idata = (guint32)bswap_32(n);                   \
357         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
358                 w_err = 1;                              \
359         wrote += 4;                                     \
360 }
361
362 #define PUT_CACHE_DATA_INT(n)                           \
363 {                                                       \
364         walk_data[0]=(((guint32)n)&0x000000ff);                 \
365         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
366         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
367         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
368         walk_data += 4;                                 \
369         wrote += 4;                                     \
370 }
371
372 #define WRITE_CACHE_DATA(data, fp) \
373 { \
374         size_t len;                                     \
375         if (data == NULL)                               \
376                 len = 0;                                \
377         else                                            \
378                 len = strlen(data);                     \
379         WRITE_CACHE_DATA_INT(len, fp);                  \
380         if (w_err == 0 && len > 0) {                    \
381                 if (SC_FWRITE(data, 1, len, fp) != len) \
382                         w_err = 1;                      \
383                 wrote += len;                           \
384         } \
385 }
386
387 #define PUT_CACHE_DATA(data)                            \
388 {                                                       \
389         size_t len;                                     \
390         if (data == NULL)                               \
391                 len = 0;                                \
392         else                                            \
393                 len = strlen(data);                     \
394         PUT_CACHE_DATA_INT(len);                        \
395         if (len > 0) {                                  \
396                 memcpy(walk_data, data, len);           \
397                 walk_data += len;                       \
398                 wrote += len;                           \
399         }                                               \
400 }
401
402 static FILE *msgcache_open_data_file(const gchar *file, guint version,
403                                      DataOpenMode mode,
404                                      gchar *buf, size_t buf_size)
405 {
406         FILE *fp;
407         gint32 data_ver;
408
409         cm_return_val_if_fail(file != NULL, NULL);
410
411         if (mode == DATA_WRITE) {
412                 int w_err = 0, wrote = 0;
413                 if ((fp = g_fopen(file, "wb")) == NULL) {
414                         FILE_OP_ERROR(file, "fopen");
415                         return NULL;
416                 }
417                 if (change_file_mode_rw(fp, file) < 0)
418                         FILE_OP_ERROR(file, "chmod");
419
420                 WRITE_CACHE_DATA_INT(version, fp);
421                 if (w_err != 0) {
422                         g_warning("failed to write int\n");
423                         fclose(fp);
424                         return NULL;
425                 }
426                 return fp;
427         }
428
429         /* check version */
430         if ((fp = g_fopen(file, "rb")) == NULL)
431                 debug_print("Mark/Cache file '%s' not found\n", file);
432         else {
433                 if (buf && buf_size > 0)
434                         setvbuf(fp, buf, _IOFBF, buf_size);
435                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
436                          version != bswap_32(data_ver)) {
437                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
438                                   file, bswap_32(data_ver), version);
439                         fclose(fp);
440                         fp = NULL;
441                 }
442                 data_ver = bswap_32(data_ver);
443         }
444         
445         if (mode == DATA_READ)
446                 return fp;
447
448         if (fp) {
449                 /* reopen with append mode */
450                 fclose(fp);
451                 if ((fp = g_fopen(file, "ab")) == NULL)
452                         FILE_OP_ERROR(file, "fopen");
453         } else {
454                 /* open with overwrite mode if mark file doesn't exist or
455                    version is different */
456                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
457                                             buf_size);
458         }
459
460         return fp;
461 }
462
463 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
464                                          StringConverter *conv)
465 {
466         gchar *tmpstr = NULL;
467         size_t ni;
468         guint32 len;
469
470         *str = NULL;
471         if (!swapping) {
472                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
473                     len > G_MAXINT) {
474                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
475                                   "of %zd bytes at offset %ld\n", ni, sizeof(len), 
476                                   ftell(fp));
477                         return -1;
478                 }
479         } else {
480                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
481                     bswap_32(len) > G_MAXINT) {
482                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
483                                   "of %zd bytes at offset %ld\n", ni, sizeof(len), 
484                                   ftell(fp));
485                         return -1;
486                 }
487                 len = bswap_32(len);
488         }
489
490         if (len == 0)
491                 return 0;
492
493         tmpstr = g_try_malloc(len + 1);
494
495         if(tmpstr == NULL) {
496                 return -1;
497         }
498
499         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
500                 g_warning("read_data_str: Cache data corrupted, read %zd of %u "
501                           "bytes at offset %ld\n", 
502                           ni, len, ftell(fp));
503                 g_free(tmpstr);
504                 return -1;
505         }
506         tmpstr[len] = 0;
507
508         if (conv != NULL) {
509                 *str = conv->convert(conv, tmpstr);
510                 g_free(tmpstr);
511         } else 
512                 *str = tmpstr;
513
514         return len;
515 }
516
517 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
518                                          StringConverter *conv)
519 {
520         gchar *tmpstr = NULL;
521
522         *str = NULL;
523
524         if (len == 0)
525                 return 0;
526
527         if(len > 2*1024*1024) {
528                 g_warning("read_data_str: refusing to allocate %d bytes.\n", len);
529                 return -1;
530         }
531
532         tmpstr = g_try_malloc(len + 1);
533
534         if(tmpstr == NULL) {
535                 return -1;
536         }
537
538         memcpy(tmpstr, src, len);
539         tmpstr[len] = 0;
540
541         if (conv != NULL) {
542                 *str = conv->convert(conv, tmpstr);
543                 g_free(tmpstr);
544         } else 
545                 *str = tmpstr;
546
547         return len;
548 }
549
550 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
551 {
552         CharsetConverter *charsetconv = (CharsetConverter *) conv;
553
554         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
555 }
556
557 static void strconv_charset_free(StringConverter *conv)
558 {
559         CharsetConverter *charsetconv = (CharsetConverter *) conv;
560
561         g_free(charsetconv->srccharset);
562         g_free(charsetconv->dstcharset);
563 }
564
565 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
566 {
567         MsgCache *cache;
568         FILE *fp;
569         MsgInfo *msginfo;
570         MsgTmpFlags tmp_flags = 0;
571         gchar file_buf[BUFFSIZE];
572         guint32 num;
573         guint refnum;
574         gboolean error = FALSE;
575         StringConverter *conv = NULL;
576         gchar *srccharset = NULL;
577         const gchar *dstcharset = NULL;
578         gchar *ref = NULL;
579         guint memusage = 0;
580         gint tmp_len = 0, map_len = -1;
581         char *cache_data = NULL;
582         struct stat st;
583
584         cm_return_val_if_fail(cache_file != NULL, NULL);
585         cm_return_val_if_fail(item != NULL, NULL);
586
587         swapping = TRUE;
588
589         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
590          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
591          * it means it's the old version (not little-endian) on a big-endian machine. The code has
592          * no effect on x86 as their file doesn't change. */
593
594         if ((fp = msgcache_open_data_file
595                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
596                 if ((fp = msgcache_open_data_file
597                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
598                         return NULL;
599                 else
600                         swapping = FALSE;
601         }
602
603         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
604
605         if (folder_has_parent_of_type(item, F_QUEUE)) {
606                 tmp_flags |= MSG_QUEUED;
607         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
608                 tmp_flags |= MSG_DRAFT;
609         }
610
611         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0) {
612                 fclose(fp);
613                 return NULL;
614         }
615         dstcharset = CS_UTF_8;
616         if (srccharset == NULL || dstcharset == NULL) {
617                 conv = NULL;
618         } else if (strcmp(srccharset, dstcharset) == 0) {
619                 debug_print("using Noop Converter\n");
620
621                 conv = NULL;
622         } else {
623                 CharsetConverter *charsetconv;
624
625                 debug_print("using CharsetConverter\n");
626
627                 charsetconv = g_new0(CharsetConverter, 1);
628                 charsetconv->converter.convert = strconv_charset_convert;
629                 charsetconv->converter.free = strconv_charset_free;
630                 charsetconv->srccharset = g_strdup(srccharset);
631                 charsetconv->dstcharset = g_strdup(dstcharset);
632
633                 conv = (StringConverter *) charsetconv;
634         }
635         g_free(srccharset);
636
637         cache = msgcache_new();
638
639         if (msgcache_use_mmap_read == TRUE) {
640                 if (fstat(fileno(fp), &st) >= 0)
641                         map_len = st.st_size;
642                 else
643                         map_len = -1;
644                 if (map_len > 0) {
645 #ifdef G_OS_WIN32
646                         cache_data = NULL;
647                         HANDLE hFile, hMapping;
648                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
649                         if (hFile == (HANDLE) -1)
650                                 goto w32_fail;
651                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
652                         if (!hMapping)
653                                 goto w32_fail;
654                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
655                         CloseHandle (hMapping);
656                 w32_fail:
657                         ;
658 #else
659                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
660 #endif
661                 }
662         } else {
663                 cache_data = NULL;
664         }
665         if (cache_data != NULL && cache_data != MAP_FAILED) {
666                 int rem_len = map_len-ftell(fp);
667                 char *walk_data = cache_data+ftell(fp);
668
669                 while(rem_len > 0) {
670                         GET_CACHE_DATA_INT(num);
671                         
672                         msginfo = procmsg_msginfo_new();
673                         msginfo->msgnum = num;
674                         memusage += sizeof(MsgInfo);
675
676                         GET_CACHE_DATA_INT(msginfo->size);
677                         GET_CACHE_DATA_INT(msginfo->mtime);
678                         GET_CACHE_DATA_INT(msginfo->date_t);
679                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
680
681                         GET_CACHE_DATA(msginfo->fromname, memusage);
682
683                         GET_CACHE_DATA(msginfo->date, memusage);
684                         GET_CACHE_DATA(msginfo->from, memusage);
685                         GET_CACHE_DATA(msginfo->to, memusage);
686                         GET_CACHE_DATA(msginfo->cc, memusage);
687                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
688                         GET_CACHE_DATA(msginfo->subject, memusage);
689                         GET_CACHE_DATA(msginfo->msgid, memusage);
690                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
691                         GET_CACHE_DATA(msginfo->xref, memusage);
692
693                         GET_CACHE_DATA_INT(msginfo->planned_download);
694                         GET_CACHE_DATA_INT(msginfo->total_size);
695                         GET_CACHE_DATA_INT(refnum);
696
697                         for (; refnum != 0; refnum--) {
698                                 ref = NULL;
699
700                                 GET_CACHE_DATA(ref, memusage);
701
702                                 if (ref && *ref)
703                                         msginfo->references =
704                                                 g_slist_prepend(msginfo->references, ref);
705                         }
706                         if (msginfo->references)
707                                 msginfo->references =
708                                         g_slist_reverse(msginfo->references);
709
710                         msginfo->folder = item;
711                         msginfo->flags.tmp_flags |= tmp_flags;
712
713                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
714                         if(msginfo->msgid)
715                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
716                 }
717         } else {
718                 while (fread(&num, sizeof(num), 1, fp) == 1) {
719                         if (swapping)
720                                 num = bswap_32(num);
721
722                         msginfo = procmsg_msginfo_new();
723                         msginfo->msgnum = num;
724                         memusage += sizeof(MsgInfo);
725
726                         READ_CACHE_DATA_INT(msginfo->size, fp);
727                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
728                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
729                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
730
731                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
732
733                         READ_CACHE_DATA(msginfo->date, fp, memusage);
734                         READ_CACHE_DATA(msginfo->from, fp, memusage);
735                         READ_CACHE_DATA(msginfo->to, fp, memusage);
736                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
737                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
738                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
739                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
740                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
741                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
742
743                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
744                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
745                         READ_CACHE_DATA_INT(refnum, fp);
746
747                         for (; refnum != 0; refnum--) {
748                                 ref = NULL;
749
750                                 READ_CACHE_DATA(ref, fp, memusage);
751
752                                 if (ref && *ref)
753                                         msginfo->references =
754                                                 g_slist_prepend(msginfo->references, ref);
755                         }
756                         if (msginfo->references)
757                                 msginfo->references =
758                                         g_slist_reverse(msginfo->references);
759
760                         msginfo->folder = item;
761                         msginfo->flags.tmp_flags |= tmp_flags;
762
763                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
764                         if(msginfo->msgid)
765                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
766                 }
767         }
768 bail_err:
769         if (cache_data != NULL && cache_data != MAP_FAILED) {
770 #ifdef G_OS_WIN32
771                 UnmapViewOfFile((void*) cache_data);
772 #else
773                 munmap(cache_data, map_len);
774 #endif
775         }
776         fclose(fp);
777         if (conv != NULL) {
778                 if (conv->free != NULL)
779                         conv->free(conv);
780                 g_free(conv);
781         }
782
783         if(error) {
784                 msgcache_destroy(cache);
785                 return NULL;
786         }
787
788         cache->last_access = time(NULL);
789         cache->memusage = memusage;
790
791         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
792         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
793
794         return cache;
795 }
796
797 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
798 {
799         FILE *fp;
800         MsgInfo *msginfo;
801         MsgPermFlags perm_flags;
802         guint32 num;
803         gint map_len = -1;
804         char *cache_data = NULL;
805         struct stat st;
806         gboolean error = FALSE;
807
808         swapping = TRUE;
809
810         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
811          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
812          * it means it's the old version (not little-endian) on a big-endian machine. The code has
813          * no effect on x86 as their file doesn't change. */
814
815         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
816                 /* see if it isn't swapped ? */
817                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
818                         return;
819                 else
820                         swapping = FALSE; /* yay */
821         }
822         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
823         
824         if (msgcache_use_mmap_read) {
825                 if (fstat(fileno(fp), &st) >= 0)
826                         map_len = st.st_size;
827                 else
828                         map_len = -1;
829                 if (map_len > 0) {
830 #ifdef G_OS_WIN32
831                         cache_data = NULL;
832                         HANDLE hFile, hMapping;
833                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
834                         if (hFile == (HANDLE) -1)
835                                 goto w32_fail2;
836                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
837                         if (!hMapping)
838                                 goto w32_fail2;
839                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
840                         CloseHandle (hMapping);
841                 w32_fail2:
842                         ;
843 #else
844                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
845 #endif
846                 }
847         } else {
848                 cache_data = NULL;
849         }
850         if (cache_data != NULL && cache_data != MAP_FAILED) {
851                 int rem_len = map_len-ftell(fp);
852                 char *walk_data = cache_data+ftell(fp);
853
854                 while(rem_len > 0) {
855                         GET_CACHE_DATA_INT(num);
856                         GET_CACHE_DATA_INT(perm_flags);
857                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
858                         if(msginfo) {
859                                 msginfo->flags.perm_flags = perm_flags;
860                         }
861                 }
862         } else {
863                 while (fread(&num, sizeof(num), 1, fp) == 1) {
864                         if (swapping)
865                                 num = bswap_32(num);
866                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) {
867                                 error = TRUE;
868                                 break;
869                         }
870                         if (swapping)
871                                 perm_flags = bswap_32(perm_flags);
872                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
873                         if(msginfo) {
874                                 msginfo->flags.perm_flags = perm_flags;
875                         }
876                 }       
877         }
878 bail_err:
879         if (cache_data != NULL && cache_data != MAP_FAILED) {
880 #ifdef G_OS_WIN32
881                 UnmapViewOfFile((void*) cache_data);
882 #else
883                 munmap(cache_data, map_len);
884 #endif
885         }
886         fclose(fp);
887         if (error) {
888                 debug_print("error reading cache mark from %s\n", mark_file);
889         }
890 }
891
892 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
893 {
894         FILE *fp;
895         MsgInfo *msginfo;
896         guint32 num;
897         gint map_len = -1;
898         char *cache_data = NULL;
899         struct stat st;
900         gboolean error = FALSE;
901
902         swapping = TRUE;
903
904         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
905          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
906          * it means it's the old version (not little-endian) on a big-endian machine. The code has
907          * no effect on x86 as their file doesn't change. */
908
909         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
910                 /* see if it isn't swapped ? */
911                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
912                         return;
913                 else
914                         swapping = FALSE; /* yay */
915         }
916         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
917         
918         if (msgcache_use_mmap_read) {
919                 if (fstat(fileno(fp), &st) >= 0)
920                         map_len = st.st_size;
921                 else
922                         map_len = -1;
923                 if (map_len > 0) {
924 #ifdef G_OS_WIN32
925                         cache_data = NULL;
926                         HANDLE hFile, hMapping;
927                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
928                         if (hFile == (HANDLE) -1)
929                                 goto w32_fail6;
930                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
931                         if (!hMapping)
932                                 goto w32_fail6;
933                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
934                         CloseHandle (hMapping);
935                 w32_fail6:
936                         ;
937 #else
938                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
939 #endif
940                 }
941         } else {
942                 cache_data = NULL;
943         }
944         if (cache_data != NULL && cache_data != MAP_FAILED) {
945                 int rem_len = map_len-ftell(fp);
946                 char *walk_data = cache_data+ftell(fp);
947
948                 while(rem_len > 0) {
949                         gint id = -1;
950                         GET_CACHE_DATA_INT(num);
951                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
952                         if(msginfo) {
953                                 g_slist_free(msginfo->tags);
954                                 msginfo->tags = NULL;
955                                 do {
956                                         GET_CACHE_DATA_INT(id);
957                                         if (id > 0) {
958                                                 msginfo->tags = g_slist_prepend(
959                                                         msginfo->tags, 
960                                                         GINT_TO_POINTER(id));
961                                         }
962                                 } while (id > 0);
963                                 msginfo->tags = g_slist_reverse(msginfo->tags);
964                         }
965                 }
966         } else {
967                 while (fread(&num, sizeof(num), 1, fp) == 1) {
968                         gint id = -1;
969                         if (swapping)
970                                 num = bswap_32(num);
971                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
972                         if(msginfo) {
973                                 g_slist_free(msginfo->tags);
974                                 msginfo->tags = NULL;
975                                 do {
976                                         if (fread(&id, sizeof(id), 1, fp) != 1) 
977                                                 id = -1;
978                                         if (swapping)
979                                                 id = bswap_32(id);
980                                         if (id > 0) {
981                                                 msginfo->tags = g_slist_prepend(
982                                                         msginfo->tags, 
983                                                         GINT_TO_POINTER(id));
984                                         }
985                                 } while (id > 0);
986                                 msginfo->tags = g_slist_reverse(msginfo->tags);
987                         }
988                 }
989         }
990 bail_err:
991         if (cache_data != NULL && cache_data != MAP_FAILED) {
992 #ifdef G_OS_WIN32
993                 UnmapViewOfFile((void*) cache_data);
994 #else
995                 munmap(cache_data, map_len);
996 #endif
997         }
998         fclose(fp);
999         if (error) {
1000                 debug_print("error reading cache tags from %s\n", tags_file);
1001         }
1002 }
1003
1004 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
1005 {
1006         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
1007         GSList *cur;
1008         int w_err = 0, wrote = 0;
1009
1010         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1011         WRITE_CACHE_DATA_INT(msginfo->size, fp);
1012         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
1013         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
1014         WRITE_CACHE_DATA_INT(flags, fp);
1015
1016         WRITE_CACHE_DATA(msginfo->fromname, fp);
1017
1018         WRITE_CACHE_DATA(msginfo->date, fp);
1019         WRITE_CACHE_DATA(msginfo->from, fp);
1020         WRITE_CACHE_DATA(msginfo->to, fp);
1021         WRITE_CACHE_DATA(msginfo->cc, fp);
1022         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
1023         WRITE_CACHE_DATA(msginfo->subject, fp);
1024         WRITE_CACHE_DATA(msginfo->msgid, fp);
1025         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
1026         WRITE_CACHE_DATA(msginfo->xref, fp);
1027         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
1028         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
1029         
1030         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
1031
1032         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
1033                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
1034         }
1035         return w_err ? -1 : wrote;
1036 }
1037
1038 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
1039 {
1040         MsgPermFlags flags = msginfo->flags.perm_flags;
1041         int w_err = 0, wrote = 0;
1042         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1043         WRITE_CACHE_DATA_INT(flags, fp);
1044         return w_err ? -1 : wrote;
1045 }
1046
1047 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
1048 {
1049         GSList *cur = msginfo->tags;
1050         int w_err = 0, wrote = 0;
1051
1052         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1053         for (; cur; cur = cur->next) {
1054                 gint id = GPOINTER_TO_INT(cur->data);
1055                 if (tags_get_tag(id) != NULL) {
1056                         WRITE_CACHE_DATA_INT(id, fp);
1057                 }
1058         }
1059         WRITE_CACHE_DATA_INT(-1, fp);
1060
1061         return w_err ? -1 : wrote;
1062 }
1063
1064 struct write_fps
1065 {
1066         FILE *cache_fp;
1067         FILE *mark_fp;
1068         FILE *tags_fp;
1069         int error;
1070         guint cache_size;
1071         guint mark_size;
1072         guint tags_size;
1073 };
1074
1075 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1076 {
1077         MsgInfo *msginfo;
1078         struct write_fps *write_fps;
1079         int tmp;
1080
1081         msginfo = (MsgInfo *)value;
1082         write_fps = user_data;
1083
1084         if (write_fps->cache_fp) {
1085                 tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1086                 if (tmp < 0)
1087                         write_fps->error = 1;
1088                 else
1089                         write_fps->cache_size += tmp;
1090         }
1091         if (write_fps->mark_fp) {
1092         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1093                 if (tmp < 0)
1094                         write_fps->error = 1;
1095                 else
1096                         write_fps->mark_size += tmp;
1097                 }
1098         if (write_fps->tags_fp) {
1099                 tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1100                 if (tmp < 0)
1101                         write_fps->error = 1;
1102                 else
1103                         write_fps->tags_size += tmp;
1104         }
1105 }
1106
1107 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1108 {
1109         struct write_fps write_fps;
1110         gchar *new_cache, *new_mark, *new_tags;
1111         int w_err = 0, wrote = 0;
1112
1113         START_TIMING("");
1114         cm_return_val_if_fail(cache != NULL, -1);
1115
1116         new_cache = g_strconcat(cache_file, ".new", NULL);
1117         new_mark  = g_strconcat(mark_file, ".new", NULL);
1118         new_tags  = g_strconcat(tags_file, ".new", NULL);
1119
1120         write_fps.error = 0;
1121         write_fps.cache_size = 0;
1122         write_fps.mark_size = 0;
1123         write_fps.tags_size = 0;
1124
1125         /* open files and write headers */
1126
1127         if (cache_file) {
1128                 write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1129                         DATA_WRITE, NULL, 0);
1130                 if (write_fps.cache_fp == NULL) {
1131                         g_free(new_cache);
1132                         g_free(new_mark);
1133                         g_free(new_tags);
1134                         return -1;
1135                 }
1136                 WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1137         } else {
1138                 write_fps.cache_fp = NULL;
1139         }
1140
1141         if (w_err != 0) {
1142                 g_warning("failed to write charset\n");
1143                 if (write_fps.cache_fp)
1144                         fclose(write_fps.cache_fp);
1145                 claws_unlink(new_cache);
1146                 g_free(new_cache);
1147                 g_free(new_mark);
1148                 g_free(new_tags);
1149                 return -1;
1150         }
1151
1152         if (mark_file) {
1153                 write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1154                         DATA_WRITE, NULL, 0);
1155                 if (write_fps.mark_fp == NULL) {
1156                         if (write_fps.cache_fp)
1157                                 fclose(write_fps.cache_fp);
1158                         claws_unlink(new_cache);
1159                         g_free(new_cache);
1160                         g_free(new_mark);
1161                         g_free(new_tags);
1162                         return -1;
1163                 }
1164         } else {
1165                 write_fps.mark_fp = NULL;
1166         }
1167
1168         if (tags_file) {
1169                 write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1170                         DATA_WRITE, NULL, 0);
1171                 if (write_fps.tags_fp == NULL) {
1172                         if (write_fps.cache_fp)
1173                                 fclose(write_fps.cache_fp);
1174                         if (write_fps.mark_fp)
1175                                 fclose(write_fps.mark_fp);
1176                         claws_unlink(new_cache);
1177                         claws_unlink(new_mark);
1178                         g_free(new_cache);
1179                         g_free(new_mark);
1180                         g_free(new_tags);
1181                         return -1;
1182                 }
1183         } else {
1184                 write_fps.tags_fp = NULL;
1185         }
1186
1187         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1188
1189         if (write_fps.cache_fp && change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1190                 FILE_OP_ERROR(new_cache, "chmod");
1191
1192         /* headers written, note file size */
1193         if (write_fps.cache_fp)
1194                 write_fps.cache_size = ftell(write_fps.cache_fp);
1195         if (write_fps.mark_fp)
1196                 write_fps.mark_size = ftell(write_fps.mark_fp);
1197         if (write_fps.tags_fp)
1198                 write_fps.tags_size = ftell(write_fps.tags_fp);
1199
1200 #ifdef HAVE_FWRITE_UNLOCKED
1201         /* lock files for write once (instead of once per fwrite) */
1202         if (write_fps.cache_fp)
1203                 flockfile(write_fps.cache_fp);
1204         if (write_fps.mark_fp)
1205                 flockfile(write_fps.mark_fp);
1206         if (write_fps.tags_fp)
1207                 flockfile(write_fps.tags_fp);
1208 #endif
1209         /* write data to the files */
1210         g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1211 #ifdef HAVE_FWRITE_UNLOCKED
1212         /* unlock files */
1213         if (write_fps.cache_fp)
1214                 funlockfile(write_fps.cache_fp);
1215         if (write_fps.mark_fp)
1216                 funlockfile(write_fps.mark_fp);
1217         if (write_fps.tags_fp)
1218                 funlockfile(write_fps.tags_fp);
1219 #endif
1220         /* flush buffers */
1221         if (write_fps.cache_fp)
1222                 write_fps.error |= (fflush(write_fps.cache_fp) != 0);
1223         if (write_fps.mark_fp)
1224                 write_fps.error |= (fflush(write_fps.mark_fp) != 0);
1225         if (write_fps.tags_fp)
1226                 write_fps.error |= (fflush(write_fps.tags_fp) != 0);
1227
1228         /* sync to filesystem */
1229         if (prefs_common.flush_metadata && write_fps.cache_fp)
1230                 write_fps.error |= (fsync(fileno(write_fps.cache_fp)) != 0);
1231         if (prefs_common.flush_metadata && write_fps.mark_fp)
1232                 write_fps.error |= (fsync(fileno(write_fps.mark_fp)) != 0);
1233         if (prefs_common.flush_metadata && write_fps.tags_fp)
1234                 write_fps.error |= (fsync(fileno(write_fps.tags_fp)) != 0);
1235
1236         /* close files */
1237         if (write_fps.cache_fp)
1238                 write_fps.error |= (fclose(write_fps.cache_fp) != 0);
1239         if (write_fps.mark_fp)
1240                 write_fps.error |= (fclose(write_fps.mark_fp) != 0);
1241         if (write_fps.tags_fp)
1242                 write_fps.error |= (fclose(write_fps.tags_fp) != 0);
1243
1244
1245         if (write_fps.error != 0) {
1246                 /* in case of error, forget all */
1247                 claws_unlink(new_cache);
1248                 claws_unlink(new_mark);
1249                 claws_unlink(new_tags);
1250                 g_free(new_cache);
1251                 g_free(new_mark);
1252                 g_free(new_tags);
1253                 return -1;
1254         } else {
1255                 /* switch files */
1256                 if (cache_file)
1257                         move_file(new_cache, cache_file, TRUE);
1258                 if (mark_file)
1259                         move_file(new_mark, mark_file, TRUE);
1260                 if (tags_file)
1261                         move_file(new_tags, tags_file, TRUE);
1262                 cache->last_access = time(NULL);
1263         }
1264
1265         g_free(new_cache);
1266         g_free(new_mark);
1267         g_free(new_tags);
1268         debug_print("done.\n");
1269         END_TIMING();
1270         return 0;
1271 }
1272