fix CID 1596595: Resource leaks, and CID 1596594: (CHECKED_RETURN)
[claws.git] / src / msgcache.c
1 /*
2  * Claws Mail -- a GTK based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2012 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  * 
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #include "claws-features.h"
23 #endif
24
25 #include "defs.h"
26
27 #include <stdio.h>
28
29 #include <glib.h>
30 #include <glib/gi18n.h>
31 #ifdef _WIN32
32 # define MAP_FAILED     ((char *) -1)
33 #else
34 # include <sys/mman.h>
35 #endif
36 #include <sys/types.h>
37 #include <sys/stat.h>
38
39 #include <time.h>
40
41 #include "msgcache.h"
42 #include "utils.h"
43 #include "procmsg.h"
44 #include "codeconv.h"
45 #include "timing.h"
46 #include "tags.h"
47 #include "prefs_common.h"
48 #include "file-utils.h"
49
50 #if G_BYTE_ORDER == G_BIG_ENDIAN
51 #define bswap_32(x) \
52      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
53       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
54      
55 #define MMAP_TO_GUINT32(x)      \
56         (((x[3]&0xff)) |        \
57          ((x[2]&0xff) << 8) |   \
58          ((x[1]&0xff) << 16) |  \
59          ((x[0]&0xff) << 24))
60
61 #define MMAP_TO_GUINT32_SWAPPED(x)      \
62         (((x[0]&0xff)) |                \
63          ((x[1]&0xff) << 8) |           \
64          ((x[2]&0xff) << 16) |          \
65          ((x[3]&0xff) << 24))
66
67 static gboolean msgcache_use_mmap_read = TRUE;
68
69 #else
70 #define bswap_32(x) (x)
71
72 #define MMAP_TO_GUINT32(x)      \
73         (((x[0]&0xff)) |        \
74          ((x[1]&0xff) << 8) |   \
75          ((x[2]&0xff) << 16) |  \
76          ((x[3]&0xff) << 24))
77
78 #define MMAP_TO_GUINT32_SWAPPED(x)      \
79         (((x[0]&0xff)) |                \
80          ((x[1]&0xff) << 8) |           \
81          ((x[2]&0xff) << 16) |          \
82          ((x[3]&0xff) << 24))
83
84 static gboolean msgcache_use_mmap_read = TRUE;
85 #endif
86
87 static gboolean swapping = TRUE;
88
89 typedef enum
90 {
91         DATA_READ,
92         DATA_WRITE,
93         DATA_APPEND
94 } DataOpenMode;
95
96 struct _MsgCache {
97         GHashTable      *msgnum_table;
98         GHashTable      *msgid_table;
99         guint            memusage;
100         time_t           last_access;
101 };
102
103 typedef struct _StringConverter StringConverter;
104 struct _StringConverter {
105         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
106         void   (*free)    (StringConverter *converter);
107 };
108
109 typedef struct _StrdupConverter StrdupConverter;
110 struct _StrdupConverter {
111         StringConverter converter;
112 };
113
114 typedef struct _CharsetConverter CharsetConverter;
115 struct _CharsetConverter {
116         StringConverter converter;
117
118         gchar *srccharset;
119         gchar *dstcharset;
120 };
121
122 MsgCache *msgcache_new(void)
123 {
124         MsgCache *cache;
125         
126         cache = g_new0(MsgCache, 1),
127         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
128         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
129         cache->last_access = time(NULL);
130
131         return cache;
132 }
133
134 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
135 {
136         procmsg_msginfo_free((MsgInfo **)&msginfo);
137         return TRUE;
138 }                                                                                         
139
140 void msgcache_destroy(MsgCache *cache)
141 {
142         cm_return_if_fail(cache != NULL);
143
144         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
145         g_hash_table_destroy(cache->msgid_table);
146         g_hash_table_destroy(cache->msgnum_table);
147         g_free(cache);
148 }
149
150 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
151 {
152         MsgInfo *newmsginfo;
153
154         cm_return_if_fail(cache != NULL);
155         cm_return_if_fail(msginfo != NULL);
156
157         newmsginfo = procmsg_msginfo_new_ref(msginfo);
158         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
159         if(newmsginfo->msgid != NULL)
160                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
161         cache->memusage += procmsg_msginfo_memusage(msginfo);
162         cache->last_access = time(NULL);
163
164         msginfo->folder->cache_dirty = TRUE;
165
166         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
167 }
168
169 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
170 {
171         MsgInfo *msginfo;
172
173         cm_return_if_fail(cache != NULL);
174
175         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
176         if(!msginfo)
177                 return;
178
179         cache->memusage -= procmsg_msginfo_memusage(msginfo);
180         if(msginfo->msgid)
181                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
182         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
183
184         msginfo->folder->cache_dirty = TRUE;
185
186         procmsg_msginfo_free(&msginfo);
187         cache->last_access = time(NULL);
188
189
190         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
191 }
192
193 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
194 {
195         MsgInfo *oldmsginfo, *newmsginfo;
196         
197         cm_return_if_fail(cache != NULL);
198         cm_return_if_fail(msginfo != NULL);
199
200         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
201         if(oldmsginfo && oldmsginfo->msgid) 
202                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
203         if (oldmsginfo) {
204                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
205                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
206                 procmsg_msginfo_free(&oldmsginfo);
207         }
208
209         newmsginfo = procmsg_msginfo_new_ref(msginfo);
210         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
211         if(newmsginfo->msgid)
212                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
213         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
214         cache->last_access = time(NULL);
215         
216         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
217
218         msginfo->folder->cache_dirty = TRUE;
219
220         return;
221 }
222
223 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
224 {
225         MsgInfo *msginfo;
226
227         cm_return_val_if_fail(cache != NULL, NULL);
228
229         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
230         if(!msginfo)
231                 return NULL;
232         cache->last_access = time(NULL);
233         
234         return procmsg_msginfo_new_ref(msginfo);
235 }
236
237 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
238 {
239         MsgInfo *msginfo;
240         
241         cm_return_val_if_fail(cache != NULL, NULL);
242         cm_return_val_if_fail(msgid != NULL, NULL);
243
244         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
245         if(!msginfo)
246                 return NULL;
247         cache->last_access = time(NULL);
248         
249         return procmsg_msginfo_new_ref(msginfo);        
250 }
251
252 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
253 {
254         MsgInfoList **listptr = user_data;
255         MsgInfo *msginfo = value;
256
257         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
258 }
259
260 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
261 {
262         MsgInfoList *msg_list = NULL;
263         START_TIMING("");
264         cm_return_val_if_fail(cache != NULL, NULL);
265
266         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
267         cache->last_access = time(NULL);
268         
269         msg_list = g_slist_reverse(msg_list);
270         END_TIMING();
271         return msg_list;
272 }
273
274 time_t msgcache_get_last_access_time(MsgCache *cache)
275 {
276         cm_return_val_if_fail(cache != NULL, 0);
277         
278         return cache->last_access;
279 }
280
281 gint msgcache_get_memory_usage(MsgCache *cache)
282 {
283         cm_return_val_if_fail(cache != NULL, 0);
284
285         return cache->memusage;
286 }
287
288 /*
289  *  Cache saving functions
290  */
291
292 #define READ_CACHE_DATA(data, fp, total_len) \
293 { \
294         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
295                 procmsg_msginfo_free(&msginfo); \
296                 error = TRUE; \
297                 goto bail_err; \
298         } \
299         total_len += tmp_len; \
300 }
301
302 #define READ_CACHE_DATA_INT(n, fp) \
303 { \
304         guint32 idata; \
305         size_t ni; \
306  \
307         if ((ni = claws_fread(&idata, sizeof(idata), 1, fp)) != 1) { \
308                 g_warning("read_int: cache data corrupted, read %"G_GSIZE_FORMAT" of %"G_GSIZE_FORMAT" at " \
309                           "offset %ld", ni, sizeof(idata), ftell(fp)); \
310                 procmsg_msginfo_free(&msginfo); \
311                 error = TRUE; \
312                 goto bail_err; \
313         } else \
314                 n = swapping ? bswap_32(idata) : (idata);\
315 }
316
317 #define GET_CACHE_DATA_INT(n)                                                                   \
318 {                                                                                               \
319         if (rem_len < 4) {                                                                      \
320                 g_print("error at rem_len:%d\n", rem_len);                                      \
321                 procmsg_msginfo_free(&msginfo); \
322                 error = TRUE;                                                                   \
323                 goto bail_err;                                                                  \
324         }                                                                                       \
325         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data)));     \
326         walk_data += 4; rem_len -= 4;                                                           \
327 }
328
329 #define GET_CACHE_DATA(data, total_len) \
330 { \
331         GET_CACHE_DATA_INT(tmp_len);    \
332         if (rem_len < tmp_len) {                                                                \
333                 g_print("error at rem_len:%d (tmp_len %d)\n", rem_len, tmp_len);                \
334                 procmsg_msginfo_free(&msginfo); \
335                 error = TRUE;                                                                   \
336                 goto bail_err;                                                                  \
337         }                                                                                       \
338         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
339                 g_print("error at rem_len:%d\n", rem_len);\
340                 procmsg_msginfo_free(&msginfo); \
341                 error = TRUE; \
342                 goto bail_err; \
343         } \
344         total_len += tmp_len; \
345         walk_data += tmp_len; rem_len -= tmp_len; \
346 }
347
348
349 #define WRITE_CACHE_DATA_INT(n, fp)                     \
350 {                                                       \
351         guint32 idata;                                  \
352                                                         \
353         idata = (guint32)bswap_32(n);                   \
354         if (claws_fwrite(&idata, sizeof(idata), 1, fp) != 1)    \
355                 w_err = 1;                              \
356         wrote += 4;                                     \
357 }
358
359 #define PUT_CACHE_DATA_INT(n)                           \
360 {                                                       \
361         walk_data[0]=(((guint32)n)&0x000000ff);                 \
362         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
363         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
364         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
365         walk_data += 4;                                 \
366         wrote += 4;                                     \
367 }
368
369 #define WRITE_CACHE_DATA(data, fp) \
370 { \
371         size_t len;                                     \
372         if (data == NULL)                               \
373                 len = 0;                                \
374         else                                            \
375                 len = strlen(data);                     \
376         WRITE_CACHE_DATA_INT(len, fp);                  \
377         if (w_err == 0 && len > 0) {                    \
378                 if (claws_fwrite(data, 1, len, fp) != len)      \
379                         w_err = 1;                      \
380                 wrote += len;                           \
381         } \
382 }
383
384 #define PUT_CACHE_DATA(data)                            \
385 {                                                       \
386         size_t len;                                     \
387         if (data == NULL)                               \
388                 len = 0;                                \
389         else                                            \
390                 len = strlen(data);                     \
391         PUT_CACHE_DATA_INT(len);                        \
392         if (len > 0) {                                  \
393                 memcpy(walk_data, data, len);           \
394                 walk_data += len;                       \
395                 wrote += len;                           \
396         }                                               \
397 }
398
399 static FILE *msgcache_open_data_file(const gchar *file, guint version,
400                                      DataOpenMode mode,
401                                      gchar *buf, size_t buf_size)
402 {
403         FILE *fp;
404         gint32 data_ver = 0;
405
406         cm_return_val_if_fail(file != NULL, NULL);
407
408         if (mode == DATA_WRITE) {
409                 int w_err = 0, wrote = 0;
410                 if ((fp = claws_fopen(file, "wb")) == NULL) {
411                         FILE_OP_ERROR(file, "claws_fopen");
412                         return NULL;
413                 }
414                 if (change_file_mode_rw(fp, file) < 0)
415                         FILE_OP_ERROR(file, "chmod");
416
417                 WRITE_CACHE_DATA_INT(version, fp);
418                 if (w_err != 0) {
419                         g_warning("failed to write int");
420                         claws_fclose(fp);
421                         return NULL;
422                 }
423                 return fp;
424         }
425
426         /* check version */
427         if ((fp = claws_fopen(file, "rb")) == NULL)
428                 debug_print("Mark/Cache file '%s' not found\n", file);
429         else {
430                 if (buf && buf_size > 0)
431                         setvbuf(fp, buf, _IOFBF, buf_size);
432                 if (claws_fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
433                          version != bswap_32(data_ver)) {
434                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
435                                   file, bswap_32(data_ver), version);
436                         claws_fclose(fp);
437                         fp = NULL;
438                 }
439                 data_ver = bswap_32(data_ver);
440         }
441         
442         if (mode == DATA_READ)
443                 return fp;
444
445         if (fp) {
446                 /* reopen with append mode */
447                 claws_fclose(fp);
448                 if ((fp = claws_fopen(file, "ab")) == NULL)
449                         FILE_OP_ERROR(file, "claws_fopen");
450         } else {
451                 /* open with overwrite mode if mark file doesn't exist or
452                    version is different */
453                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
454                                             buf_size);
455         }
456
457         return fp;
458 }
459
460 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
461                                          StringConverter *conv)
462 {
463         gchar *tmpstr = NULL;
464         size_t ni;
465         guint32 len;
466
467         *str = NULL;
468         if (!swapping) {
469                 if ((ni = claws_fread(&len, sizeof(len), 1, fp) != 1) ||
470                     len > G_MAXINT) {
471                         g_warning("read_data_str: cache data (len) corrupted, read %"G_GSIZE_FORMAT
472                                   " of %"G_GSIZE_FORMAT" bytes at offset %ld", ni, sizeof(len),
473                                   ftell(fp));
474                         return -1;
475                 }
476         } else {
477                 if ((ni = claws_fread(&len, sizeof(len), 1, fp) != 1) ||
478                     bswap_32(len) > G_MAXINT) {
479                         g_warning("read_data_str: cache data (len) corrupted, read %"G_GSIZE_FORMAT
480                                   " of %"G_GSIZE_FORMAT" bytes at offset %ld", ni, sizeof(len),
481                                   ftell(fp));
482                         return -1;
483                 }
484                 len = bswap_32(len);
485         }
486
487         if (len == 0)
488                 return 0;
489
490         tmpstr = g_try_malloc(len + 1);
491
492         if(tmpstr == NULL) {
493                 return -1;
494         }
495
496         if ((ni = claws_fread(tmpstr, 1, len, fp)) != len) {
497                 g_warning("read_data_str: cache data corrupted, read %"G_GSIZE_FORMAT" of %u "
498                           "bytes at offset %ld",
499                           ni, len, ftell(fp));
500                 g_free(tmpstr);
501                 return -1;
502         }
503         tmpstr[len] = 0;
504
505         if (conv != NULL) {
506                 *str = conv->convert(conv, tmpstr);
507                 g_free(tmpstr);
508         } else 
509                 *str = tmpstr;
510
511         return len;
512 }
513
514 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
515                                          StringConverter *conv)
516 {
517         gchar *tmpstr = NULL;
518
519         *str = NULL;
520
521         if (len == 0)
522                 return 0;
523
524         if(len > 2*1024*1024) {
525                 g_warning("read_data_str: refusing to allocate %d bytes", len);
526                 return -1;
527         }
528
529         tmpstr = g_try_malloc(len + 1);
530
531         if(tmpstr == NULL) {
532                 return -1;
533         }
534
535         memcpy(tmpstr, src, len);
536         tmpstr[len] = 0;
537
538         if (conv != NULL) {
539                 *str = conv->convert(conv, tmpstr);
540                 g_free(tmpstr);
541         } else 
542                 *str = tmpstr;
543
544         return len;
545 }
546
547 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
548 {
549         CharsetConverter *charsetconv = (CharsetConverter *) conv;
550
551         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
552 }
553
554 static void strconv_charset_free(StringConverter *conv)
555 {
556         CharsetConverter *charsetconv = (CharsetConverter *) conv;
557
558         g_free(charsetconv->srccharset);
559         g_free(charsetconv->dstcharset);
560 }
561
562 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
563 {
564         MsgCache *cache;
565         FILE *fp;
566         MsgInfo *msginfo;
567         MsgTmpFlags tmp_flags = 0;
568         gchar file_buf[BUFFSIZE];
569         guint32 num;
570         guint refnum;
571         gboolean error = FALSE;
572         StringConverter *conv = NULL;
573         gchar *srccharset = NULL;
574         const gchar *dstcharset = NULL;
575         gchar *ref = NULL;
576         guint memusage = 0;
577         gint tmp_len = 0, map_len = -1;
578         char *cache_data = NULL;
579         struct stat st;
580
581         cm_return_val_if_fail(cache_file != NULL, NULL);
582         cm_return_val_if_fail(item != NULL, NULL);
583
584         swapping = TRUE;
585
586         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
587          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
588          * it means it's the old version (not little-endian) on a big-endian machine. The code has
589          * no effect on x86 as their file doesn't change. */
590
591         if ((fp = msgcache_open_data_file
592                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
593                 if ((fp = msgcache_open_data_file
594                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
595                         return NULL;
596                 else
597                         swapping = FALSE;
598         }
599
600         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
601
602         if (folder_has_parent_of_type(item, F_QUEUE)) {
603                 tmp_flags |= MSG_QUEUED;
604         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
605                 tmp_flags |= MSG_DRAFT;
606         }
607
608         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0) {
609                 claws_fclose(fp);
610                 return NULL;
611         }
612         dstcharset = CS_UTF_8;
613         if (srccharset == NULL || dstcharset == NULL) {
614                 conv = NULL;
615         } else if (strcmp(srccharset, dstcharset) == 0) {
616                 debug_print("using Noop Converter\n");
617
618                 conv = NULL;
619         } else {
620                 CharsetConverter *charsetconv;
621
622                 debug_print("using CharsetConverter\n");
623
624                 charsetconv = g_new0(CharsetConverter, 1);
625                 charsetconv->converter.convert = strconv_charset_convert;
626                 charsetconv->converter.free = strconv_charset_free;
627                 charsetconv->srccharset = g_strdup(srccharset);
628                 charsetconv->dstcharset = g_strdup(dstcharset);
629
630                 conv = (StringConverter *) charsetconv;
631         }
632         g_free(srccharset);
633
634         cache = msgcache_new();
635
636         if (msgcache_use_mmap_read == TRUE) {
637                 if (fstat(fileno(fp), &st) >= 0)
638                         map_len = st.st_size;
639                 else
640                         map_len = -1;
641                 if (map_len > 0) {
642 #ifdef G_OS_WIN32
643                         cache_data = NULL;
644                         HANDLE hFile, hMapping;
645                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
646                         if (hFile == (HANDLE) -1)
647                                 goto w32_fail;
648                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
649                         if (!hMapping)
650                                 goto w32_fail;
651                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
652                         CloseHandle (hMapping);
653                 w32_fail:
654                         ;
655 #else
656                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
657 #endif
658                 }
659         } else {
660                 cache_data = NULL;
661         }
662         if (cache_data != NULL && cache_data != MAP_FAILED) {
663                 int rem_len = map_len-ftell(fp);
664                 char *walk_data = cache_data+ftell(fp);
665
666                 while(rem_len > 0) {
667                         msginfo = procmsg_msginfo_new();
668
669                         GET_CACHE_DATA_INT(num);
670
671                         msginfo->msgnum = num;
672                         memusage += sizeof(MsgInfo);
673
674                         GET_CACHE_DATA_INT(msginfo->size);
675                         GET_CACHE_DATA_INT(msginfo->mtime);
676                         GET_CACHE_DATA_INT(msginfo->date_t);
677                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
678
679                         GET_CACHE_DATA(msginfo->fromname, memusage);
680
681                         GET_CACHE_DATA(msginfo->date, memusage);
682                         GET_CACHE_DATA(msginfo->from, memusage);
683                         GET_CACHE_DATA(msginfo->to, memusage);
684                         GET_CACHE_DATA(msginfo->cc, memusage);
685                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
686                         GET_CACHE_DATA(msginfo->subject, memusage);
687                         GET_CACHE_DATA(msginfo->msgid, memusage);
688                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
689                         GET_CACHE_DATA(msginfo->xref, memusage);
690
691                         GET_CACHE_DATA_INT(msginfo->planned_download);
692                         GET_CACHE_DATA_INT(msginfo->total_size);
693                         GET_CACHE_DATA_INT(refnum);
694
695                         for (; refnum != 0; refnum--) {
696                                 ref = NULL;
697
698                                 GET_CACHE_DATA(ref, memusage);
699
700                                 if (ref && *ref)
701                                         msginfo->references =
702                                                 g_slist_prepend(msginfo->references, ref);
703                         }
704                         if (msginfo->references)
705                                 msginfo->references =
706                                         g_slist_reverse(msginfo->references);
707
708                         msginfo->folder = item;
709                         msginfo->flags.tmp_flags |= tmp_flags;
710
711                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
712                         if(msginfo->msgid)
713                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
714                 }
715         } else {
716                 while (claws_fread(&num, sizeof(num), 1, fp) == 1) {
717                         if (swapping)
718                                 num = bswap_32(num);
719
720                         msginfo = procmsg_msginfo_new();
721                         msginfo->msgnum = num;
722                         memusage += sizeof(MsgInfo);
723
724                         READ_CACHE_DATA_INT(msginfo->size, fp);
725                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
726                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
727                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
728
729                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
730
731                         READ_CACHE_DATA(msginfo->date, fp, memusage);
732                         READ_CACHE_DATA(msginfo->from, fp, memusage);
733                         READ_CACHE_DATA(msginfo->to, fp, memusage);
734                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
735                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
736                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
737                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
738                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
739                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
740
741                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
742                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
743                         READ_CACHE_DATA_INT(refnum, fp);
744
745                         for (; refnum != 0; refnum--) {
746                                 ref = NULL;
747
748                                 READ_CACHE_DATA(ref, fp, memusage);
749
750                                 if (ref && *ref)
751                                         msginfo->references =
752                                                 g_slist_prepend(msginfo->references, ref);
753                         }
754                         if (msginfo->references)
755                                 msginfo->references =
756                                         g_slist_reverse(msginfo->references);
757
758                         msginfo->folder = item;
759                         msginfo->flags.tmp_flags |= tmp_flags;
760
761                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
762                         if(msginfo->msgid)
763                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
764                 }
765         }
766 bail_err:
767         if (cache_data != NULL && cache_data != MAP_FAILED) {
768 #ifdef G_OS_WIN32
769                 UnmapViewOfFile((void*) cache_data);
770 #else
771                 munmap(cache_data, map_len);
772 #endif
773         }
774         claws_fclose(fp);
775         if (conv != NULL) {
776                 if (conv->free != NULL)
777                         conv->free(conv);
778                 g_free(conv);
779         }
780
781         if(error) {
782                 msgcache_destroy(cache);
783                 return NULL;
784         }
785
786         cache->last_access = time(NULL);
787         cache->memusage = memusage;
788
789         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
790         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
791
792         return cache;
793 }
794
795 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
796 {
797         FILE *fp;
798         MsgInfo *msginfo;
799         MsgPermFlags perm_flags;
800         guint32 num;
801         gint map_len = -1;
802         char *cache_data = NULL;
803         struct stat st;
804         gboolean error = FALSE;
805
806         swapping = TRUE;
807
808         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
809          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
810          * it means it's the old version (not little-endian) on a big-endian machine. The code has
811          * no effect on x86 as their file doesn't change. */
812
813         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
814                 /* see if it isn't swapped ? */
815                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
816                         return;
817                 else
818                         swapping = FALSE; /* yay */
819         }
820         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
821         
822         if (msgcache_use_mmap_read) {
823                 if (fstat(fileno(fp), &st) >= 0)
824                         map_len = st.st_size;
825                 else
826                         map_len = -1;
827                 if (map_len > 0) {
828 #ifdef G_OS_WIN32
829                         cache_data = NULL;
830                         HANDLE hFile, hMapping;
831                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
832                         if (hFile == (HANDLE) -1)
833                                 goto w32_fail2;
834                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
835                         if (!hMapping)
836                                 goto w32_fail2;
837                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
838                         CloseHandle (hMapping);
839                 w32_fail2:
840                         ;
841 #else
842                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
843 #endif
844                 }
845         } else {
846                 cache_data = NULL;
847         }
848         if (cache_data != NULL && cache_data != MAP_FAILED) {
849                 int rem_len = map_len-ftell(fp);
850                 char *walk_data = cache_data+ftell(fp);
851
852                 while(rem_len > 0) {
853                         GET_CACHE_DATA_INT(num);
854                         GET_CACHE_DATA_INT(perm_flags);
855                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
856                         if(msginfo) {
857                                 msginfo->flags.perm_flags = perm_flags;
858                         }
859                 }
860         } else {
861                 while (claws_fread(&num, sizeof(num), 1, fp) == 1) {
862                         if (swapping)
863                                 num = bswap_32(num);
864                         if (claws_fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) {
865                                 error = TRUE;
866                                 break;
867                         }
868                         if (swapping)
869                                 perm_flags = bswap_32(perm_flags);
870                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
871                         if(msginfo) {
872                                 msginfo->flags.perm_flags = perm_flags;
873                         }
874                 }       
875         }
876 bail_err:
877         if (cache_data != NULL && cache_data != MAP_FAILED) {
878 #ifdef G_OS_WIN32
879                 UnmapViewOfFile((void*) cache_data);
880 #else
881                 munmap(cache_data, map_len);
882 #endif
883         }
884         claws_fclose(fp);
885         if (error) {
886                 debug_print("error reading cache mark from %s\n", mark_file);
887         }
888 }
889
890 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
891 {
892         FILE *fp;
893         MsgInfo *msginfo;
894         guint32 num;
895         gint map_len = -1;
896         char *cache_data = NULL;
897         struct stat st;
898         gboolean error = FALSE;
899
900         swapping = TRUE;
901
902         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
903          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
904          * it means it's the old version (not little-endian) on a big-endian machine. The code has
905          * no effect on x86 as their file doesn't change. */
906
907         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
908                 /* see if it isn't swapped ? */
909                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
910                         return;
911                 else
912                         swapping = FALSE; /* yay */
913         }
914         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
915         
916         if (msgcache_use_mmap_read) {
917                 if (fstat(fileno(fp), &st) >= 0)
918                         map_len = st.st_size;
919                 else
920                         map_len = -1;
921                 if (map_len > 0) {
922 #ifdef G_OS_WIN32
923                         cache_data = NULL;
924                         HANDLE hFile, hMapping;
925                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
926                         if (hFile == (HANDLE) -1)
927                                 goto w32_fail6;
928                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
929                         if (!hMapping)
930                                 goto w32_fail6;
931                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
932                         CloseHandle (hMapping);
933                 w32_fail6:
934                         ;
935 #else
936                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
937 #endif
938                 }
939         } else {
940                 cache_data = NULL;
941         }
942         if (cache_data != NULL && cache_data != MAP_FAILED) {
943                 int rem_len = map_len-ftell(fp);
944                 char *walk_data = cache_data+ftell(fp);
945
946                 while(rem_len > 0) {
947                         gint id = -1;
948                         GET_CACHE_DATA_INT(num);
949                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
950                         if(msginfo) {
951                                 g_slist_free(msginfo->tags);
952                                 msginfo->tags = NULL;
953                                 do {
954                                         GET_CACHE_DATA_INT(id);
955                                         if (id > 0) {
956                                                 msginfo->tags = g_slist_prepend(
957                                                         msginfo->tags, 
958                                                         GINT_TO_POINTER(id));
959                                         }
960                                 } while (id > 0);
961                                 msginfo->tags = g_slist_reverse(msginfo->tags);
962                         }
963                 }
964         } else {
965                 while (claws_fread(&num, sizeof(num), 1, fp) == 1) {
966                         gint id = -1;
967                         if (swapping)
968                                 num = bswap_32(num);
969                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
970                         if(msginfo) {
971                                 g_slist_free(msginfo->tags);
972                                 msginfo->tags = NULL;
973                                 do {
974                                         if (claws_fread(&id, sizeof(id), 1, fp) != 1) 
975                                                 id = -1;
976                                         if (swapping)
977                                                 id = bswap_32(id);
978                                         if (id > 0) {
979                                                 msginfo->tags = g_slist_prepend(
980                                                         msginfo->tags, 
981                                                         GINT_TO_POINTER(id));
982                                         }
983                                 } while (id > 0);
984                                 msginfo->tags = g_slist_reverse(msginfo->tags);
985                         }
986                 }
987         }
988 bail_err:
989         if (cache_data != NULL && cache_data != MAP_FAILED) {
990 #ifdef G_OS_WIN32
991                 UnmapViewOfFile((void*) cache_data);
992 #else
993                 munmap(cache_data, map_len);
994 #endif
995         }
996         claws_fclose(fp);
997         if (error) {
998                 debug_print("error reading cache tags from %s\n", tags_file);
999         }
1000 }
1001
1002 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
1003 {
1004         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
1005         GSList *cur;
1006         int w_err = 0, wrote = 0;
1007
1008         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1009         WRITE_CACHE_DATA_INT(msginfo->size, fp);
1010         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
1011         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
1012         WRITE_CACHE_DATA_INT(flags, fp);
1013
1014         WRITE_CACHE_DATA(msginfo->fromname, fp);
1015
1016         WRITE_CACHE_DATA(msginfo->date, fp);
1017         WRITE_CACHE_DATA(msginfo->from, fp);
1018         WRITE_CACHE_DATA(msginfo->to, fp);
1019         WRITE_CACHE_DATA(msginfo->cc, fp);
1020         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
1021         WRITE_CACHE_DATA(msginfo->subject, fp);
1022         WRITE_CACHE_DATA(msginfo->msgid, fp);
1023         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
1024         WRITE_CACHE_DATA(msginfo->xref, fp);
1025         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
1026         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
1027         
1028         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
1029
1030         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
1031                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
1032         }
1033         return w_err ? -1 : wrote;
1034 }
1035
1036 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
1037 {
1038         MsgPermFlags flags = msginfo->flags.perm_flags;
1039         int w_err = 0, wrote = 0;
1040         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1041         WRITE_CACHE_DATA_INT(flags, fp);
1042         return w_err ? -1 : wrote;
1043 }
1044
1045 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
1046 {
1047         GSList *cur = msginfo->tags;
1048         int w_err = 0, wrote = 0;
1049
1050         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1051         for (; cur; cur = cur->next) {
1052                 gint id = GPOINTER_TO_INT(cur->data);
1053                 if (tags_get_tag(id) != NULL) {
1054                         WRITE_CACHE_DATA_INT(id, fp);
1055                 }
1056         }
1057         WRITE_CACHE_DATA_INT(-1, fp);
1058
1059         return w_err ? -1 : wrote;
1060 }
1061
1062 struct write_fps
1063 {
1064         FILE *cache_fp;
1065         FILE *mark_fp;
1066         FILE *tags_fp;
1067         int error;
1068         guint cache_size;
1069         guint mark_size;
1070         guint tags_size;
1071 };
1072
1073 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1074 {
1075         MsgInfo *msginfo;
1076         struct write_fps *write_fps;
1077         int tmp;
1078
1079         msginfo = (MsgInfo *)value;
1080         write_fps = user_data;
1081
1082         if (write_fps->cache_fp) {
1083                 tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1084                 if (tmp < 0)
1085                         write_fps->error = 1;
1086                 else
1087                         write_fps->cache_size += tmp;
1088         }
1089         if (write_fps->mark_fp) {
1090         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1091                 if (tmp < 0)
1092                         write_fps->error = 1;
1093                 else
1094                         write_fps->mark_size += tmp;
1095                 }
1096         if (write_fps->tags_fp) {
1097                 tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1098                 if (tmp < 0)
1099                         write_fps->error = 1;
1100                 else
1101                         write_fps->tags_size += tmp;
1102         }
1103 }
1104
1105 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1106 {
1107         struct write_fps write_fps;
1108         gchar *new_cache = NULL, *new_mark = NULL, *new_tags = NULL;
1109         int w_err = 0, wrote = 0;
1110
1111         START_TIMING("");
1112         cm_return_val_if_fail(cache != NULL, -1);
1113
1114         if (cache_file)
1115                 new_cache = g_strconcat(cache_file, ".new", NULL);
1116         if (mark_file)
1117                 new_mark  = g_strconcat(mark_file, ".new", NULL);
1118         if (tags_file)
1119                 new_tags  = g_strconcat(tags_file, ".new", NULL);
1120
1121         write_fps.error = 0;
1122         write_fps.cache_size = 0;
1123         write_fps.mark_size = 0;
1124         write_fps.tags_size = 0;
1125
1126         /* open files and write headers */
1127
1128         if (cache_file) {
1129                 write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1130                         DATA_WRITE, NULL, 0);
1131                 if (write_fps.cache_fp == NULL) {
1132                         g_free(new_cache);
1133                         g_free(new_mark);
1134                         g_free(new_tags);
1135                         return -1;
1136                 }
1137                 WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1138         } else {
1139                 write_fps.cache_fp = NULL;
1140         }
1141
1142         if (w_err != 0) {
1143                 g_warning("failed to write charset");
1144                 if (write_fps.cache_fp)
1145                         claws_fclose(write_fps.cache_fp);
1146                 claws_unlink(new_cache);
1147                 g_free(new_cache);
1148                 g_free(new_mark);
1149                 g_free(new_tags);
1150                 return -1;
1151         }
1152
1153         if (mark_file) {
1154                 write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1155                         DATA_WRITE, NULL, 0);
1156                 if (write_fps.mark_fp == NULL) {
1157                         if (write_fps.cache_fp)
1158                                 claws_fclose(write_fps.cache_fp);
1159                         claws_unlink(new_cache);
1160                         g_free(new_cache);
1161                         g_free(new_mark);
1162                         g_free(new_tags);
1163                         return -1;
1164                 }
1165         } else {
1166                 write_fps.mark_fp = NULL;
1167         }
1168
1169         if (tags_file) {
1170                 write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1171                         DATA_WRITE, NULL, 0);
1172                 if (write_fps.tags_fp == NULL) {
1173                         if (write_fps.cache_fp)
1174                                 claws_fclose(write_fps.cache_fp);
1175                         if (write_fps.mark_fp)
1176                                 claws_fclose(write_fps.mark_fp);
1177                         claws_unlink(new_cache);
1178                         claws_unlink(new_mark);
1179                         g_free(new_cache);
1180                         g_free(new_mark);
1181                         g_free(new_tags);
1182                         return -1;
1183                 }
1184         } else {
1185                 write_fps.tags_fp = NULL;
1186         }
1187
1188         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1189
1190         if (write_fps.cache_fp && change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1191                 FILE_OP_ERROR(new_cache, "chmod");
1192
1193         /* headers written, note file size */
1194         if (write_fps.cache_fp)
1195                 write_fps.cache_size = ftell(write_fps.cache_fp);
1196         if (write_fps.mark_fp)
1197                 write_fps.mark_size = ftell(write_fps.mark_fp);
1198         if (write_fps.tags_fp)
1199                 write_fps.tags_size = ftell(write_fps.tags_fp);
1200
1201         /* write data to the files */
1202         g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1203
1204         /* close files */
1205         if (write_fps.cache_fp)
1206                 write_fps.error |= (claws_safe_fclose(write_fps.cache_fp) != 0);
1207         if (write_fps.mark_fp)
1208                 write_fps.error |= (claws_safe_fclose(write_fps.mark_fp) != 0);
1209         if (write_fps.tags_fp)
1210                 write_fps.error |= (claws_safe_fclose(write_fps.tags_fp) != 0);
1211
1212
1213         if (write_fps.error != 0) {
1214                 /* in case of error, forget all */
1215                 claws_unlink(new_cache);
1216                 claws_unlink(new_mark);
1217                 claws_unlink(new_tags);
1218                 g_free(new_cache);
1219                 g_free(new_mark);
1220                 g_free(new_tags);
1221                 return -1;
1222         } else {
1223                 /* switch files */
1224                 if (cache_file)
1225                         move_file(new_cache, cache_file, TRUE);
1226                 if (mark_file)
1227                         move_file(new_mark, mark_file, TRUE);
1228                 if (tags_file)
1229                         move_file(new_tags, tags_file, TRUE);
1230                 cache->last_access = time(NULL);
1231         }
1232
1233         g_free(new_cache);
1234         g_free(new_mark);
1235         g_free(new_tags);
1236         debug_print("done.\n");
1237         END_TIMING();
1238         return 0;
1239 }
1240