Merge branch 'master' of ssh://git.claws-mail.org/home/git/claws
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2012 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  * 
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #include "claws-features.h"
23 #endif
24
25 #include "defs.h"
26
27 #include <stdio.h>
28
29 #include <glib.h>
30 #include <glib/gi18n.h>
31 #ifdef _WIN32
32 # include <w32lib.h>
33 # define MAP_FAILED     ((char *) -1)
34 #else
35 # include <sys/mman.h>
36 #endif
37 #include <sys/types.h>
38 #include <sys/stat.h>
39
40 #include <time.h>
41
42 #include "msgcache.h"
43 #include "utils.h"
44 #include "procmsg.h"
45 #include "codeconv.h"
46 #include "timing.h"
47 #include "tags.h"
48 #include "prefs_common.h"
49 #include "file-utils.h"
50
51 #if G_BYTE_ORDER == G_BIG_ENDIAN
52 #define bswap_32(x) \
53      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
54       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
55      
56 #define MMAP_TO_GUINT32(x)      \
57         (((x[3]&0xff)) |        \
58          ((x[2]&0xff) << 8) |   \
59          ((x[1]&0xff) << 16) |  \
60          ((x[0]&0xff) << 24))
61
62 #define MMAP_TO_GUINT32_SWAPPED(x)      \
63         (((x[0]&0xff)) |                \
64          ((x[1]&0xff) << 8) |           \
65          ((x[2]&0xff) << 16) |          \
66          ((x[3]&0xff) << 24))
67
68 static gboolean msgcache_use_mmap_read = TRUE;
69
70 #else
71 #define bswap_32(x) (x)
72
73 #define MMAP_TO_GUINT32(x)      \
74         (((x[0]&0xff)) |        \
75          ((x[1]&0xff) << 8) |   \
76          ((x[2]&0xff) << 16) |  \
77          ((x[3]&0xff) << 24))
78
79 #define MMAP_TO_GUINT32_SWAPPED(x)      \
80         (((x[0]&0xff)) |                \
81          ((x[1]&0xff) << 8) |           \
82          ((x[2]&0xff) << 16) |          \
83          ((x[3]&0xff) << 24))
84
85 static gboolean msgcache_use_mmap_read = TRUE;
86 #endif
87
88 static gboolean swapping = TRUE;
89
90 typedef enum
91 {
92         DATA_READ,
93         DATA_WRITE,
94         DATA_APPEND
95 } DataOpenMode;
96
97 struct _MsgCache {
98         GHashTable      *msgnum_table;
99         GHashTable      *msgid_table;
100         guint            memusage;
101         time_t           last_access;
102 };
103
104 typedef struct _StringConverter StringConverter;
105 struct _StringConverter {
106         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
107         void   (*free)    (StringConverter *converter);
108 };
109
110 typedef struct _StrdupConverter StrdupConverter;
111 struct _StrdupConverter {
112         StringConverter converter;
113 };
114
115 typedef struct _CharsetConverter CharsetConverter;
116 struct _CharsetConverter {
117         StringConverter converter;
118
119         gchar *srccharset;
120         gchar *dstcharset;
121 };
122
123 MsgCache *msgcache_new(void)
124 {
125         MsgCache *cache;
126         
127         cache = g_new0(MsgCache, 1),
128         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
129         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
130         cache->last_access = time(NULL);
131
132         return cache;
133 }
134
135 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
136 {
137         procmsg_msginfo_free((MsgInfo **)&msginfo);
138         return TRUE;
139 }                                                                                         
140
141 void msgcache_destroy(MsgCache *cache)
142 {
143         cm_return_if_fail(cache != NULL);
144
145         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
146         g_hash_table_destroy(cache->msgid_table);
147         g_hash_table_destroy(cache->msgnum_table);
148         g_free(cache);
149 }
150
151 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
152 {
153         MsgInfo *newmsginfo;
154
155         cm_return_if_fail(cache != NULL);
156         cm_return_if_fail(msginfo != NULL);
157
158         newmsginfo = procmsg_msginfo_new_ref(msginfo);
159         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
160         if(newmsginfo->msgid != NULL)
161                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
162         cache->memusage += procmsg_msginfo_memusage(msginfo);
163         cache->last_access = time(NULL);
164
165         msginfo->folder->cache_dirty = TRUE;
166
167         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
168 }
169
170 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
171 {
172         MsgInfo *msginfo;
173
174         cm_return_if_fail(cache != NULL);
175
176         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
177         if(!msginfo)
178                 return;
179
180         cache->memusage -= procmsg_msginfo_memusage(msginfo);
181         if(msginfo->msgid)
182                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
183         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
184
185         msginfo->folder->cache_dirty = TRUE;
186
187         procmsg_msginfo_free(&msginfo);
188         cache->last_access = time(NULL);
189
190
191         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
192 }
193
194 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
195 {
196         MsgInfo *oldmsginfo, *newmsginfo;
197         
198         cm_return_if_fail(cache != NULL);
199         cm_return_if_fail(msginfo != NULL);
200
201         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
202         if(oldmsginfo && oldmsginfo->msgid) 
203                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
204         if (oldmsginfo) {
205                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
206                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
207                 procmsg_msginfo_free(&oldmsginfo);
208         }
209
210         newmsginfo = procmsg_msginfo_new_ref(msginfo);
211         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
212         if(newmsginfo->msgid)
213                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
214         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
215         cache->last_access = time(NULL);
216         
217         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
218
219         msginfo->folder->cache_dirty = TRUE;
220
221         return;
222 }
223
224 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
225 {
226         MsgInfo *msginfo;
227
228         cm_return_val_if_fail(cache != NULL, NULL);
229
230         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
231         if(!msginfo)
232                 return NULL;
233         cache->last_access = time(NULL);
234         
235         return procmsg_msginfo_new_ref(msginfo);
236 }
237
238 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
239 {
240         MsgInfo *msginfo;
241         
242         cm_return_val_if_fail(cache != NULL, NULL);
243         cm_return_val_if_fail(msgid != NULL, NULL);
244
245         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
246         if(!msginfo)
247                 return NULL;
248         cache->last_access = time(NULL);
249         
250         return procmsg_msginfo_new_ref(msginfo);        
251 }
252
253 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
254 {
255         MsgInfoList **listptr = user_data;
256         MsgInfo *msginfo = value;
257
258         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
259 }
260
261 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
262 {
263         MsgInfoList *msg_list = NULL;
264         START_TIMING("");
265         cm_return_val_if_fail(cache != NULL, NULL);
266
267         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
268         cache->last_access = time(NULL);
269         
270         msg_list = g_slist_reverse(msg_list);
271         END_TIMING();
272         return msg_list;
273 }
274
275 time_t msgcache_get_last_access_time(MsgCache *cache)
276 {
277         cm_return_val_if_fail(cache != NULL, 0);
278         
279         return cache->last_access;
280 }
281
282 gint msgcache_get_memory_usage(MsgCache *cache)
283 {
284         cm_return_val_if_fail(cache != NULL, 0);
285
286         return cache->memusage;
287 }
288
289 /*
290  *  Cache saving functions
291  */
292
293 #define READ_CACHE_DATA(data, fp, total_len) \
294 { \
295         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
296                 procmsg_msginfo_free(&msginfo); \
297                 error = TRUE; \
298                 goto bail_err; \
299         } \
300         total_len += tmp_len; \
301 }
302
303 #define READ_CACHE_DATA_INT(n, fp) \
304 { \
305         guint32 idata; \
306         size_t ni; \
307  \
308         if ((ni = claws_fread(&idata, sizeof(idata), 1, fp)) != 1) { \
309                 g_warning("read_int: Cache data corrupted, read %zd of %zd at " \
310                           "offset %ld", ni, sizeof(idata), ftell(fp)); \
311                 procmsg_msginfo_free(&msginfo); \
312                 error = TRUE; \
313                 goto bail_err; \
314         } else \
315                 n = swapping ? bswap_32(idata) : (idata);\
316 }
317
318 #define GET_CACHE_DATA_INT(n)                                                                   \
319 {                                                                                               \
320         if (rem_len < 4) {                                                                      \
321                 g_print("error at rem_len:%d\n", rem_len);                                      \
322                 error = TRUE;                                                                   \
323                 goto bail_err;                                                                  \
324         }                                                                                       \
325         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data)));     \
326         walk_data += 4; rem_len -= 4;                                                           \
327 }
328
329 #define GET_CACHE_DATA(data, total_len) \
330 { \
331         GET_CACHE_DATA_INT(tmp_len);    \
332         if (rem_len < tmp_len) {                                                                \
333                 g_print("error at rem_len:%d (tmp_len %d)\n", rem_len, tmp_len);                \
334                 error = TRUE;                                                                   \
335                 goto bail_err;                                                                  \
336         }                                                                                       \
337         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
338                 g_print("error at rem_len:%d\n", rem_len);\
339                 procmsg_msginfo_free(&msginfo); \
340                 error = TRUE; \
341                 goto bail_err; \
342         } \
343         total_len += tmp_len; \
344         walk_data += tmp_len; rem_len -= tmp_len; \
345 }
346
347
348 #define WRITE_CACHE_DATA_INT(n, fp)                     \
349 {                                                       \
350         guint32 idata;                                  \
351                                                         \
352         idata = (guint32)bswap_32(n);                   \
353         if (claws_fwrite(&idata, sizeof(idata), 1, fp) != 1)    \
354                 w_err = 1;                              \
355         wrote += 4;                                     \
356 }
357
358 #define PUT_CACHE_DATA_INT(n)                           \
359 {                                                       \
360         walk_data[0]=(((guint32)n)&0x000000ff);                 \
361         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
362         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
363         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
364         walk_data += 4;                                 \
365         wrote += 4;                                     \
366 }
367
368 #define WRITE_CACHE_DATA(data, fp) \
369 { \
370         size_t len;                                     \
371         if (data == NULL)                               \
372                 len = 0;                                \
373         else                                            \
374                 len = strlen(data);                     \
375         WRITE_CACHE_DATA_INT(len, fp);                  \
376         if (w_err == 0 && len > 0) {                    \
377                 if (claws_fwrite(data, 1, len, fp) != len)      \
378                         w_err = 1;                      \
379                 wrote += len;                           \
380         } \
381 }
382
383 #define PUT_CACHE_DATA(data)                            \
384 {                                                       \
385         size_t len;                                     \
386         if (data == NULL)                               \
387                 len = 0;                                \
388         else                                            \
389                 len = strlen(data);                     \
390         PUT_CACHE_DATA_INT(len);                        \
391         if (len > 0) {                                  \
392                 memcpy(walk_data, data, len);           \
393                 walk_data += len;                       \
394                 wrote += len;                           \
395         }                                               \
396 }
397
398 static FILE *msgcache_open_data_file(const gchar *file, guint version,
399                                      DataOpenMode mode,
400                                      gchar *buf, size_t buf_size)
401 {
402         FILE *fp;
403         gint32 data_ver;
404
405         cm_return_val_if_fail(file != NULL, NULL);
406
407         if (mode == DATA_WRITE) {
408                 int w_err = 0, wrote = 0;
409                 if ((fp = claws_fopen(file, "wb")) == NULL) {
410                         FILE_OP_ERROR(file, "claws_fopen");
411                         return NULL;
412                 }
413                 if (change_file_mode_rw(fp, file) < 0)
414                         FILE_OP_ERROR(file, "chmod");
415
416                 WRITE_CACHE_DATA_INT(version, fp);
417                 if (w_err != 0) {
418                         g_warning("failed to write int");
419                         claws_fclose(fp);
420                         return NULL;
421                 }
422                 return fp;
423         }
424
425         /* check version */
426         if ((fp = claws_fopen(file, "rb")) == NULL)
427                 debug_print("Mark/Cache file '%s' not found\n", file);
428         else {
429                 if (buf && buf_size > 0)
430                         setvbuf(fp, buf, _IOFBF, buf_size);
431                 if (claws_fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
432                          version != bswap_32(data_ver)) {
433                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
434                                   file, bswap_32(data_ver), version);
435                         claws_fclose(fp);
436                         fp = NULL;
437                 }
438                 data_ver = bswap_32(data_ver);
439         }
440         
441         if (mode == DATA_READ)
442                 return fp;
443
444         if (fp) {
445                 /* reopen with append mode */
446                 claws_fclose(fp);
447                 if ((fp = claws_fopen(file, "ab")) == NULL)
448                         FILE_OP_ERROR(file, "claws_fopen");
449         } else {
450                 /* open with overwrite mode if mark file doesn't exist or
451                    version is different */
452                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
453                                             buf_size);
454         }
455
456         return fp;
457 }
458
459 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
460                                          StringConverter *conv)
461 {
462         gchar *tmpstr = NULL;
463         size_t ni;
464         guint32 len;
465
466         *str = NULL;
467         if (!swapping) {
468                 if ((ni = claws_fread(&len, sizeof(len), 1, fp) != 1) ||
469                     len > G_MAXINT) {
470                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
471                                   "of %zd bytes at offset %ld", ni, sizeof(len),
472                                   ftell(fp));
473                         return -1;
474                 }
475         } else {
476                 if ((ni = claws_fread(&len, sizeof(len), 1, fp) != 1) ||
477                     bswap_32(len) > G_MAXINT) {
478                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
479                                   "of %zd bytes at offset %ld", ni, sizeof(len),
480                                   ftell(fp));
481                         return -1;
482                 }
483                 len = bswap_32(len);
484         }
485
486         if (len == 0)
487                 return 0;
488
489         tmpstr = g_try_malloc(len + 1);
490
491         if(tmpstr == NULL) {
492                 return -1;
493         }
494
495         if ((ni = claws_fread(tmpstr, 1, len, fp)) != len) {
496                 g_warning("read_data_str: Cache data corrupted, read %zd of %u "
497                           "bytes at offset %ld",
498                           ni, len, ftell(fp));
499                 g_free(tmpstr);
500                 return -1;
501         }
502         tmpstr[len] = 0;
503
504         if (conv != NULL) {
505                 *str = conv->convert(conv, tmpstr);
506                 g_free(tmpstr);
507         } else 
508                 *str = tmpstr;
509
510         return len;
511 }
512
513 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
514                                          StringConverter *conv)
515 {
516         gchar *tmpstr = NULL;
517
518         *str = NULL;
519
520         if (len == 0)
521                 return 0;
522
523         if(len > 2*1024*1024) {
524                 g_warning("read_data_str: refusing to allocate %d bytes.", len);
525                 return -1;
526         }
527
528         tmpstr = g_try_malloc(len + 1);
529
530         if(tmpstr == NULL) {
531                 return -1;
532         }
533
534         memcpy(tmpstr, src, len);
535         tmpstr[len] = 0;
536
537         if (conv != NULL) {
538                 *str = conv->convert(conv, tmpstr);
539                 g_free(tmpstr);
540         } else 
541                 *str = tmpstr;
542
543         return len;
544 }
545
546 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
547 {
548         CharsetConverter *charsetconv = (CharsetConverter *) conv;
549
550         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
551 }
552
553 static void strconv_charset_free(StringConverter *conv)
554 {
555         CharsetConverter *charsetconv = (CharsetConverter *) conv;
556
557         g_free(charsetconv->srccharset);
558         g_free(charsetconv->dstcharset);
559 }
560
561 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
562 {
563         MsgCache *cache;
564         FILE *fp;
565         MsgInfo *msginfo;
566         MsgTmpFlags tmp_flags = 0;
567         gchar file_buf[BUFFSIZE];
568         guint32 num;
569         guint refnum;
570         gboolean error = FALSE;
571         StringConverter *conv = NULL;
572         gchar *srccharset = NULL;
573         const gchar *dstcharset = NULL;
574         gchar *ref = NULL;
575         guint memusage = 0;
576         gint tmp_len = 0, map_len = -1;
577         char *cache_data = NULL;
578         struct stat st;
579
580         cm_return_val_if_fail(cache_file != NULL, NULL);
581         cm_return_val_if_fail(item != NULL, NULL);
582
583         swapping = TRUE;
584
585         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
586          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
587          * it means it's the old version (not little-endian) on a big-endian machine. The code has
588          * no effect on x86 as their file doesn't change. */
589
590         if ((fp = msgcache_open_data_file
591                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
592                 if ((fp = msgcache_open_data_file
593                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
594                         return NULL;
595                 else
596                         swapping = FALSE;
597         }
598
599         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
600
601         if (folder_has_parent_of_type(item, F_QUEUE)) {
602                 tmp_flags |= MSG_QUEUED;
603         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
604                 tmp_flags |= MSG_DRAFT;
605         }
606
607         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0) {
608                 claws_fclose(fp);
609                 return NULL;
610         }
611         dstcharset = CS_UTF_8;
612         if (srccharset == NULL || dstcharset == NULL) {
613                 conv = NULL;
614         } else if (strcmp(srccharset, dstcharset) == 0) {
615                 debug_print("using Noop Converter\n");
616
617                 conv = NULL;
618         } else {
619                 CharsetConverter *charsetconv;
620
621                 debug_print("using CharsetConverter\n");
622
623                 charsetconv = g_new0(CharsetConverter, 1);
624                 charsetconv->converter.convert = strconv_charset_convert;
625                 charsetconv->converter.free = strconv_charset_free;
626                 charsetconv->srccharset = g_strdup(srccharset);
627                 charsetconv->dstcharset = g_strdup(dstcharset);
628
629                 conv = (StringConverter *) charsetconv;
630         }
631         g_free(srccharset);
632
633         cache = msgcache_new();
634
635         if (msgcache_use_mmap_read == TRUE) {
636                 if (fstat(fileno(fp), &st) >= 0)
637                         map_len = st.st_size;
638                 else
639                         map_len = -1;
640                 if (map_len > 0) {
641 #ifdef G_OS_WIN32
642                         cache_data = NULL;
643                         HANDLE hFile, hMapping;
644                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
645                         if (hFile == (HANDLE) -1)
646                                 goto w32_fail;
647                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
648                         if (!hMapping)
649                                 goto w32_fail;
650                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
651                         CloseHandle (hMapping);
652                 w32_fail:
653                         ;
654 #else
655                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
656 #endif
657                 }
658         } else {
659                 cache_data = NULL;
660         }
661         if (cache_data != NULL && cache_data != MAP_FAILED) {
662                 int rem_len = map_len-ftell(fp);
663                 char *walk_data = cache_data+ftell(fp);
664
665                 while(rem_len > 0) {
666                         GET_CACHE_DATA_INT(num);
667                         
668                         msginfo = procmsg_msginfo_new();
669                         msginfo->msgnum = num;
670                         memusage += sizeof(MsgInfo);
671
672                         GET_CACHE_DATA_INT(msginfo->size);
673                         GET_CACHE_DATA_INT(msginfo->mtime);
674                         GET_CACHE_DATA_INT(msginfo->date_t);
675                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
676
677                         GET_CACHE_DATA(msginfo->fromname, memusage);
678
679                         GET_CACHE_DATA(msginfo->date, memusage);
680                         GET_CACHE_DATA(msginfo->from, memusage);
681                         GET_CACHE_DATA(msginfo->to, memusage);
682                         GET_CACHE_DATA(msginfo->cc, memusage);
683                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
684                         GET_CACHE_DATA(msginfo->subject, memusage);
685                         GET_CACHE_DATA(msginfo->msgid, memusage);
686                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
687                         GET_CACHE_DATA(msginfo->xref, memusage);
688
689                         GET_CACHE_DATA_INT(msginfo->planned_download);
690                         GET_CACHE_DATA_INT(msginfo->total_size);
691                         GET_CACHE_DATA_INT(refnum);
692
693                         for (; refnum != 0; refnum--) {
694                                 ref = NULL;
695
696                                 GET_CACHE_DATA(ref, memusage);
697
698                                 if (ref && *ref)
699                                         msginfo->references =
700                                                 g_slist_prepend(msginfo->references, ref);
701                         }
702                         if (msginfo->references)
703                                 msginfo->references =
704                                         g_slist_reverse(msginfo->references);
705
706                         msginfo->folder = item;
707                         msginfo->flags.tmp_flags |= tmp_flags;
708
709                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
710                         if(msginfo->msgid)
711                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
712                 }
713         } else {
714                 while (claws_fread(&num, sizeof(num), 1, fp) == 1) {
715                         if (swapping)
716                                 num = bswap_32(num);
717
718                         msginfo = procmsg_msginfo_new();
719                         msginfo->msgnum = num;
720                         memusage += sizeof(MsgInfo);
721
722                         READ_CACHE_DATA_INT(msginfo->size, fp);
723                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
724                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
725                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
726
727                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
728
729                         READ_CACHE_DATA(msginfo->date, fp, memusage);
730                         READ_CACHE_DATA(msginfo->from, fp, memusage);
731                         READ_CACHE_DATA(msginfo->to, fp, memusage);
732                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
733                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
734                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
735                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
736                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
737                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
738
739                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
740                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
741                         READ_CACHE_DATA_INT(refnum, fp);
742
743                         for (; refnum != 0; refnum--) {
744                                 ref = NULL;
745
746                                 READ_CACHE_DATA(ref, fp, memusage);
747
748                                 if (ref && *ref)
749                                         msginfo->references =
750                                                 g_slist_prepend(msginfo->references, ref);
751                         }
752                         if (msginfo->references)
753                                 msginfo->references =
754                                         g_slist_reverse(msginfo->references);
755
756                         msginfo->folder = item;
757                         msginfo->flags.tmp_flags |= tmp_flags;
758
759                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
760                         if(msginfo->msgid)
761                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
762                 }
763         }
764 bail_err:
765         if (cache_data != NULL && cache_data != MAP_FAILED) {
766 #ifdef G_OS_WIN32
767                 UnmapViewOfFile((void*) cache_data);
768 #else
769                 munmap(cache_data, map_len);
770 #endif
771         }
772         claws_fclose(fp);
773         if (conv != NULL) {
774                 if (conv->free != NULL)
775                         conv->free(conv);
776                 g_free(conv);
777         }
778
779         if(error) {
780                 msgcache_destroy(cache);
781                 return NULL;
782         }
783
784         cache->last_access = time(NULL);
785         cache->memusage = memusage;
786
787         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
788         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
789
790         return cache;
791 }
792
793 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
794 {
795         FILE *fp;
796         MsgInfo *msginfo;
797         MsgPermFlags perm_flags;
798         guint32 num;
799         gint map_len = -1;
800         char *cache_data = NULL;
801         struct stat st;
802         gboolean error = FALSE;
803
804         swapping = TRUE;
805
806         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
807          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
808          * it means it's the old version (not little-endian) on a big-endian machine. The code has
809          * no effect on x86 as their file doesn't change. */
810
811         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
812                 /* see if it isn't swapped ? */
813                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
814                         return;
815                 else
816                         swapping = FALSE; /* yay */
817         }
818         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
819         
820         if (msgcache_use_mmap_read) {
821                 if (fstat(fileno(fp), &st) >= 0)
822                         map_len = st.st_size;
823                 else
824                         map_len = -1;
825                 if (map_len > 0) {
826 #ifdef G_OS_WIN32
827                         cache_data = NULL;
828                         HANDLE hFile, hMapping;
829                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
830                         if (hFile == (HANDLE) -1)
831                                 goto w32_fail2;
832                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
833                         if (!hMapping)
834                                 goto w32_fail2;
835                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
836                         CloseHandle (hMapping);
837                 w32_fail2:
838                         ;
839 #else
840                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
841 #endif
842                 }
843         } else {
844                 cache_data = NULL;
845         }
846         if (cache_data != NULL && cache_data != MAP_FAILED) {
847                 int rem_len = map_len-ftell(fp);
848                 char *walk_data = cache_data+ftell(fp);
849
850                 while(rem_len > 0) {
851                         GET_CACHE_DATA_INT(num);
852                         GET_CACHE_DATA_INT(perm_flags);
853                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
854                         if(msginfo) {
855                                 msginfo->flags.perm_flags = perm_flags;
856                         }
857                 }
858         } else {
859                 while (claws_fread(&num, sizeof(num), 1, fp) == 1) {
860                         if (swapping)
861                                 num = bswap_32(num);
862                         if (claws_fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) {
863                                 error = TRUE;
864                                 break;
865                         }
866                         if (swapping)
867                                 perm_flags = bswap_32(perm_flags);
868                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
869                         if(msginfo) {
870                                 msginfo->flags.perm_flags = perm_flags;
871                         }
872                 }       
873         }
874 bail_err:
875         if (cache_data != NULL && cache_data != MAP_FAILED) {
876 #ifdef G_OS_WIN32
877                 UnmapViewOfFile((void*) cache_data);
878 #else
879                 munmap(cache_data, map_len);
880 #endif
881         }
882         claws_fclose(fp);
883         if (error) {
884                 debug_print("error reading cache mark from %s\n", mark_file);
885         }
886 }
887
888 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
889 {
890         FILE *fp;
891         MsgInfo *msginfo;
892         guint32 num;
893         gint map_len = -1;
894         char *cache_data = NULL;
895         struct stat st;
896         gboolean error = FALSE;
897
898         swapping = TRUE;
899
900         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
901          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
902          * it means it's the old version (not little-endian) on a big-endian machine. The code has
903          * no effect on x86 as their file doesn't change. */
904
905         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
906                 /* see if it isn't swapped ? */
907                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
908                         return;
909                 else
910                         swapping = FALSE; /* yay */
911         }
912         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
913         
914         if (msgcache_use_mmap_read) {
915                 if (fstat(fileno(fp), &st) >= 0)
916                         map_len = st.st_size;
917                 else
918                         map_len = -1;
919                 if (map_len > 0) {
920 #ifdef G_OS_WIN32
921                         cache_data = NULL;
922                         HANDLE hFile, hMapping;
923                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
924                         if (hFile == (HANDLE) -1)
925                                 goto w32_fail6;
926                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
927                         if (!hMapping)
928                                 goto w32_fail6;
929                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
930                         CloseHandle (hMapping);
931                 w32_fail6:
932                         ;
933 #else
934                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
935 #endif
936                 }
937         } else {
938                 cache_data = NULL;
939         }
940         if (cache_data != NULL && cache_data != MAP_FAILED) {
941                 int rem_len = map_len-ftell(fp);
942                 char *walk_data = cache_data+ftell(fp);
943
944                 while(rem_len > 0) {
945                         gint id = -1;
946                         GET_CACHE_DATA_INT(num);
947                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
948                         if(msginfo) {
949                                 g_slist_free(msginfo->tags);
950                                 msginfo->tags = NULL;
951                                 do {
952                                         GET_CACHE_DATA_INT(id);
953                                         if (id > 0) {
954                                                 msginfo->tags = g_slist_prepend(
955                                                         msginfo->tags, 
956                                                         GINT_TO_POINTER(id));
957                                         }
958                                 } while (id > 0);
959                                 msginfo->tags = g_slist_reverse(msginfo->tags);
960                         }
961                 }
962         } else {
963                 while (claws_fread(&num, sizeof(num), 1, fp) == 1) {
964                         gint id = -1;
965                         if (swapping)
966                                 num = bswap_32(num);
967                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
968                         if(msginfo) {
969                                 g_slist_free(msginfo->tags);
970                                 msginfo->tags = NULL;
971                                 do {
972                                         if (claws_fread(&id, sizeof(id), 1, fp) != 1) 
973                                                 id = -1;
974                                         if (swapping)
975                                                 id = bswap_32(id);
976                                         if (id > 0) {
977                                                 msginfo->tags = g_slist_prepend(
978                                                         msginfo->tags, 
979                                                         GINT_TO_POINTER(id));
980                                         }
981                                 } while (id > 0);
982                                 msginfo->tags = g_slist_reverse(msginfo->tags);
983                         }
984                 }
985         }
986 bail_err:
987         if (cache_data != NULL && cache_data != MAP_FAILED) {
988 #ifdef G_OS_WIN32
989                 UnmapViewOfFile((void*) cache_data);
990 #else
991                 munmap(cache_data, map_len);
992 #endif
993         }
994         claws_fclose(fp);
995         if (error) {
996                 debug_print("error reading cache tags from %s\n", tags_file);
997         }
998 }
999
1000 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
1001 {
1002         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
1003         GSList *cur;
1004         int w_err = 0, wrote = 0;
1005
1006         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1007         WRITE_CACHE_DATA_INT(msginfo->size, fp);
1008         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
1009         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
1010         WRITE_CACHE_DATA_INT(flags, fp);
1011
1012         WRITE_CACHE_DATA(msginfo->fromname, fp);
1013
1014         WRITE_CACHE_DATA(msginfo->date, fp);
1015         WRITE_CACHE_DATA(msginfo->from, fp);
1016         WRITE_CACHE_DATA(msginfo->to, fp);
1017         WRITE_CACHE_DATA(msginfo->cc, fp);
1018         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
1019         WRITE_CACHE_DATA(msginfo->subject, fp);
1020         WRITE_CACHE_DATA(msginfo->msgid, fp);
1021         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
1022         WRITE_CACHE_DATA(msginfo->xref, fp);
1023         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
1024         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
1025         
1026         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
1027
1028         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
1029                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
1030         }
1031         return w_err ? -1 : wrote;
1032 }
1033
1034 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
1035 {
1036         MsgPermFlags flags = msginfo->flags.perm_flags;
1037         int w_err = 0, wrote = 0;
1038         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1039         WRITE_CACHE_DATA_INT(flags, fp);
1040         return w_err ? -1 : wrote;
1041 }
1042
1043 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
1044 {
1045         GSList *cur = msginfo->tags;
1046         int w_err = 0, wrote = 0;
1047
1048         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1049         for (; cur; cur = cur->next) {
1050                 gint id = GPOINTER_TO_INT(cur->data);
1051                 if (tags_get_tag(id) != NULL) {
1052                         WRITE_CACHE_DATA_INT(id, fp);
1053                 }
1054         }
1055         WRITE_CACHE_DATA_INT(-1, fp);
1056
1057         return w_err ? -1 : wrote;
1058 }
1059
1060 struct write_fps
1061 {
1062         FILE *cache_fp;
1063         FILE *mark_fp;
1064         FILE *tags_fp;
1065         int error;
1066         guint cache_size;
1067         guint mark_size;
1068         guint tags_size;
1069 };
1070
1071 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1072 {
1073         MsgInfo *msginfo;
1074         struct write_fps *write_fps;
1075         int tmp;
1076
1077         msginfo = (MsgInfo *)value;
1078         write_fps = user_data;
1079
1080         if (write_fps->cache_fp) {
1081                 tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1082                 if (tmp < 0)
1083                         write_fps->error = 1;
1084                 else
1085                         write_fps->cache_size += tmp;
1086         }
1087         if (write_fps->mark_fp) {
1088         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1089                 if (tmp < 0)
1090                         write_fps->error = 1;
1091                 else
1092                         write_fps->mark_size += tmp;
1093                 }
1094         if (write_fps->tags_fp) {
1095                 tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1096                 if (tmp < 0)
1097                         write_fps->error = 1;
1098                 else
1099                         write_fps->tags_size += tmp;
1100         }
1101 }
1102
1103 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1104 {
1105         struct write_fps write_fps;
1106         gchar *new_cache, *new_mark, *new_tags;
1107         int w_err = 0, wrote = 0;
1108
1109         START_TIMING("");
1110         cm_return_val_if_fail(cache != NULL, -1);
1111
1112         new_cache = g_strconcat(cache_file, ".new", NULL);
1113         new_mark  = g_strconcat(mark_file, ".new", NULL);
1114         new_tags  = g_strconcat(tags_file, ".new", NULL);
1115
1116         write_fps.error = 0;
1117         write_fps.cache_size = 0;
1118         write_fps.mark_size = 0;
1119         write_fps.tags_size = 0;
1120
1121         /* open files and write headers */
1122
1123         if (cache_file) {
1124                 write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1125                         DATA_WRITE, NULL, 0);
1126                 if (write_fps.cache_fp == NULL) {
1127                         g_free(new_cache);
1128                         g_free(new_mark);
1129                         g_free(new_tags);
1130                         return -1;
1131                 }
1132                 WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1133         } else {
1134                 write_fps.cache_fp = NULL;
1135         }
1136
1137         if (w_err != 0) {
1138                 g_warning("failed to write charset");
1139                 if (write_fps.cache_fp)
1140                         claws_fclose(write_fps.cache_fp);
1141                 claws_unlink(new_cache);
1142                 g_free(new_cache);
1143                 g_free(new_mark);
1144                 g_free(new_tags);
1145                 return -1;
1146         }
1147
1148         if (mark_file) {
1149                 write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1150                         DATA_WRITE, NULL, 0);
1151                 if (write_fps.mark_fp == NULL) {
1152                         if (write_fps.cache_fp)
1153                                 claws_fclose(write_fps.cache_fp);
1154                         claws_unlink(new_cache);
1155                         g_free(new_cache);
1156                         g_free(new_mark);
1157                         g_free(new_tags);
1158                         return -1;
1159                 }
1160         } else {
1161                 write_fps.mark_fp = NULL;
1162         }
1163
1164         if (tags_file) {
1165                 write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1166                         DATA_WRITE, NULL, 0);
1167                 if (write_fps.tags_fp == NULL) {
1168                         if (write_fps.cache_fp)
1169                                 claws_fclose(write_fps.cache_fp);
1170                         if (write_fps.mark_fp)
1171                                 claws_fclose(write_fps.mark_fp);
1172                         claws_unlink(new_cache);
1173                         claws_unlink(new_mark);
1174                         g_free(new_cache);
1175                         g_free(new_mark);
1176                         g_free(new_tags);
1177                         return -1;
1178                 }
1179         } else {
1180                 write_fps.tags_fp = NULL;
1181         }
1182
1183         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1184
1185         if (write_fps.cache_fp && change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1186                 FILE_OP_ERROR(new_cache, "chmod");
1187
1188         /* headers written, note file size */
1189         if (write_fps.cache_fp)
1190                 write_fps.cache_size = ftell(write_fps.cache_fp);
1191         if (write_fps.mark_fp)
1192                 write_fps.mark_size = ftell(write_fps.mark_fp);
1193         if (write_fps.tags_fp)
1194                 write_fps.tags_size = ftell(write_fps.tags_fp);
1195
1196         /* write data to the files */
1197         g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1198
1199         /* close files */
1200         if (write_fps.cache_fp)
1201                 write_fps.error |= (claws_safe_fclose(write_fps.cache_fp) != 0);
1202         if (write_fps.mark_fp)
1203                 write_fps.error |= (claws_safe_fclose(write_fps.mark_fp) != 0);
1204         if (write_fps.tags_fp)
1205                 write_fps.error |= (claws_safe_fclose(write_fps.tags_fp) != 0);
1206
1207
1208         if (write_fps.error != 0) {
1209                 /* in case of error, forget all */
1210                 claws_unlink(new_cache);
1211                 claws_unlink(new_mark);
1212                 claws_unlink(new_tags);
1213                 g_free(new_cache);
1214                 g_free(new_mark);
1215                 g_free(new_tags);
1216                 return -1;
1217         } else {
1218                 /* switch files */
1219                 if (cache_file)
1220                         move_file(new_cache, cache_file, TRUE);
1221                 if (mark_file)
1222                         move_file(new_mark, mark_file, TRUE);
1223                 if (tags_file)
1224                         move_file(new_tags, tags_file, TRUE);
1225                 cache->last_access = time(NULL);
1226         }
1227
1228         g_free(new_cache);
1229         g_free(new_mark);
1230         g_free(new_tags);
1231         debug_print("done.\n");
1232         END_TIMING();
1233         return 0;
1234 }
1235