8909ea7044928d2f78c0c0d355f86c437457212a
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2012 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  * 
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #include "claws-features.h"
23 #endif
24
25 #include "defs.h"
26
27 #define _GNU_SOURCE
28 #include <stdio.h>
29
30 #include <glib.h>
31 #include <glib/gi18n.h>
32 #ifdef _WIN32
33 # include <w32lib.h>
34 # define MAP_FAILED     ((char *) -1)
35 #else
36 # include <sys/mman.h>
37 #endif
38 #include <sys/types.h>
39 #include <sys/stat.h>
40
41 #include <time.h>
42
43 #include "msgcache.h"
44 #include "utils.h"
45 #include "procmsg.h"
46 #include "codeconv.h"
47 #include "timing.h"
48 #include "tags.h"
49 #include "prefs_common.h"
50 #include "claws_io.h"
51
52 #ifdef HAVE_FWRITE_UNLOCKED
53 #define SC_FWRITE fwrite_unlocked
54 #else
55 #define SC_FWRITE fwrite
56 #endif
57
58 #if G_BYTE_ORDER == G_BIG_ENDIAN
59 #define bswap_32(x) \
60      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
61       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
62      
63 #define MMAP_TO_GUINT32(x)      \
64         (((x[3]&0xff)) |        \
65          ((x[2]&0xff) << 8) |   \
66          ((x[1]&0xff) << 16) |  \
67          ((x[0]&0xff) << 24))
68
69 #define MMAP_TO_GUINT32_SWAPPED(x)      \
70         (((x[0]&0xff)) |                \
71          ((x[1]&0xff) << 8) |           \
72          ((x[2]&0xff) << 16) |          \
73          ((x[3]&0xff) << 24))
74
75 static gboolean msgcache_use_mmap_read = TRUE;
76
77 #else
78 #define bswap_32(x) (x)
79
80 #define MMAP_TO_GUINT32(x)      \
81         (((x[0]&0xff)) |        \
82          ((x[1]&0xff) << 8) |   \
83          ((x[2]&0xff) << 16) |  \
84          ((x[3]&0xff) << 24))
85
86 #define MMAP_TO_GUINT32_SWAPPED(x)      \
87         (((x[0]&0xff)) |                \
88          ((x[1]&0xff) << 8) |           \
89          ((x[2]&0xff) << 16) |          \
90          ((x[3]&0xff) << 24))
91
92 static gboolean msgcache_use_mmap_read = TRUE;
93 #endif
94
95 static gboolean swapping = TRUE;
96
97 typedef enum
98 {
99         DATA_READ,
100         DATA_WRITE,
101         DATA_APPEND
102 } DataOpenMode;
103
104 struct _MsgCache {
105         GHashTable      *msgnum_table;
106         GHashTable      *msgid_table;
107         guint            memusage;
108         time_t           last_access;
109 };
110
111 typedef struct _StringConverter StringConverter;
112 struct _StringConverter {
113         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
114         void   (*free)    (StringConverter *converter);
115 };
116
117 typedef struct _StrdupConverter StrdupConverter;
118 struct _StrdupConverter {
119         StringConverter converter;
120 };
121
122 typedef struct _CharsetConverter CharsetConverter;
123 struct _CharsetConverter {
124         StringConverter converter;
125
126         gchar *srccharset;
127         gchar *dstcharset;
128 };
129
130 MsgCache *msgcache_new(void)
131 {
132         MsgCache *cache;
133         
134         cache = g_new0(MsgCache, 1),
135         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
136         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
137         cache->last_access = time(NULL);
138
139         return cache;
140 }
141
142 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
143 {
144         procmsg_msginfo_free((MsgInfo **)&msginfo);
145         return TRUE;
146 }                                                                                         
147
148 void msgcache_destroy(MsgCache *cache)
149 {
150         cm_return_if_fail(cache != NULL);
151
152         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
153         g_hash_table_destroy(cache->msgid_table);
154         g_hash_table_destroy(cache->msgnum_table);
155         g_free(cache);
156 }
157
158 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
159 {
160         MsgInfo *newmsginfo;
161
162         cm_return_if_fail(cache != NULL);
163         cm_return_if_fail(msginfo != NULL);
164
165         newmsginfo = procmsg_msginfo_new_ref(msginfo);
166         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
167         if(newmsginfo->msgid != NULL)
168                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
169         cache->memusage += procmsg_msginfo_memusage(msginfo);
170         cache->last_access = time(NULL);
171
172         msginfo->folder->cache_dirty = TRUE;
173
174         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
175 }
176
177 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
178 {
179         MsgInfo *msginfo;
180
181         cm_return_if_fail(cache != NULL);
182
183         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
184         if(!msginfo)
185                 return;
186
187         cache->memusage -= procmsg_msginfo_memusage(msginfo);
188         if(msginfo->msgid)
189                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
190         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
191
192         msginfo->folder->cache_dirty = TRUE;
193
194         procmsg_msginfo_free(&msginfo);
195         cache->last_access = time(NULL);
196
197
198         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
199 }
200
201 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
202 {
203         MsgInfo *oldmsginfo, *newmsginfo;
204         
205         cm_return_if_fail(cache != NULL);
206         cm_return_if_fail(msginfo != NULL);
207
208         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
209         if(oldmsginfo && oldmsginfo->msgid) 
210                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
211         if (oldmsginfo) {
212                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
213                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
214                 procmsg_msginfo_free(&oldmsginfo);
215         }
216
217         newmsginfo = procmsg_msginfo_new_ref(msginfo);
218         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
219         if(newmsginfo->msgid)
220                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
221         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
222         cache->last_access = time(NULL);
223         
224         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
225
226         msginfo->folder->cache_dirty = TRUE;
227
228         return;
229 }
230
231 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
232 {
233         MsgInfo *msginfo;
234
235         cm_return_val_if_fail(cache != NULL, NULL);
236
237         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
238         if(!msginfo)
239                 return NULL;
240         cache->last_access = time(NULL);
241         
242         return procmsg_msginfo_new_ref(msginfo);
243 }
244
245 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
246 {
247         MsgInfo *msginfo;
248         
249         cm_return_val_if_fail(cache != NULL, NULL);
250         cm_return_val_if_fail(msgid != NULL, NULL);
251
252         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
253         if(!msginfo)
254                 return NULL;
255         cache->last_access = time(NULL);
256         
257         return procmsg_msginfo_new_ref(msginfo);        
258 }
259
260 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
261 {
262         MsgInfoList **listptr = user_data;
263         MsgInfo *msginfo = value;
264
265         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
266 }
267
268 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
269 {
270         MsgInfoList *msg_list = NULL;
271         START_TIMING("");
272         cm_return_val_if_fail(cache != NULL, NULL);
273
274         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
275         cache->last_access = time(NULL);
276         
277         msg_list = g_slist_reverse(msg_list);
278         END_TIMING();
279         return msg_list;
280 }
281
282 time_t msgcache_get_last_access_time(MsgCache *cache)
283 {
284         cm_return_val_if_fail(cache != NULL, 0);
285         
286         return cache->last_access;
287 }
288
289 gint msgcache_get_memory_usage(MsgCache *cache)
290 {
291         cm_return_val_if_fail(cache != NULL, 0);
292
293         return cache->memusage;
294 }
295
296 /*
297  *  Cache saving functions
298  */
299
300 #define READ_CACHE_DATA(data, fp, total_len) \
301 { \
302         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
303                 procmsg_msginfo_free(&msginfo); \
304                 error = TRUE; \
305                 goto bail_err; \
306         } \
307         total_len += tmp_len; \
308 }
309
310 #define READ_CACHE_DATA_INT(n, fp) \
311 { \
312         guint32 idata; \
313         size_t ni; \
314  \
315         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
316                 g_warning("read_int: Cache data corrupted, read %zd of %zd at " \
317                           "offset %ld", ni, sizeof(idata), ftell(fp)); \
318                 procmsg_msginfo_free(&msginfo); \
319                 error = TRUE; \
320                 goto bail_err; \
321         } else \
322                 n = swapping ? bswap_32(idata) : (idata);\
323 }
324
325 #define GET_CACHE_DATA_INT(n)                                                                   \
326 {                                                                                               \
327         if (rem_len < 4) {                                                                      \
328                 g_print("error at rem_len:%d\n", rem_len);                                      \
329                 error = TRUE;                                                                   \
330                 goto bail_err;                                                                  \
331         }                                                                                       \
332         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data)));     \
333         walk_data += 4; rem_len -= 4;                                                           \
334 }
335
336 #define GET_CACHE_DATA(data, total_len) \
337 { \
338         GET_CACHE_DATA_INT(tmp_len);    \
339         if (rem_len < tmp_len) {                                                                \
340                 g_print("error at rem_len:%d (tmp_len %d)\n", rem_len, tmp_len);                \
341                 error = TRUE;                                                                   \
342                 goto bail_err;                                                                  \
343         }                                                                                       \
344         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
345                 g_print("error at rem_len:%d\n", rem_len);\
346                 procmsg_msginfo_free(&msginfo); \
347                 error = TRUE; \
348                 goto bail_err; \
349         } \
350         total_len += tmp_len; \
351         walk_data += tmp_len; rem_len -= tmp_len; \
352 }
353
354
355 #define WRITE_CACHE_DATA_INT(n, fp)                     \
356 {                                                       \
357         guint32 idata;                                  \
358                                                         \
359         idata = (guint32)bswap_32(n);                   \
360         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
361                 w_err = 1;                              \
362         wrote += 4;                                     \
363 }
364
365 #define PUT_CACHE_DATA_INT(n)                           \
366 {                                                       \
367         walk_data[0]=(((guint32)n)&0x000000ff);                 \
368         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
369         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
370         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
371         walk_data += 4;                                 \
372         wrote += 4;                                     \
373 }
374
375 #define WRITE_CACHE_DATA(data, fp) \
376 { \
377         size_t len;                                     \
378         if (data == NULL)                               \
379                 len = 0;                                \
380         else                                            \
381                 len = strlen(data);                     \
382         WRITE_CACHE_DATA_INT(len, fp);                  \
383         if (w_err == 0 && len > 0) {                    \
384                 if (SC_FWRITE(data, 1, len, fp) != len) \
385                         w_err = 1;                      \
386                 wrote += len;                           \
387         } \
388 }
389
390 #define PUT_CACHE_DATA(data)                            \
391 {                                                       \
392         size_t len;                                     \
393         if (data == NULL)                               \
394                 len = 0;                                \
395         else                                            \
396                 len = strlen(data);                     \
397         PUT_CACHE_DATA_INT(len);                        \
398         if (len > 0) {                                  \
399                 memcpy(walk_data, data, len);           \
400                 walk_data += len;                       \
401                 wrote += len;                           \
402         }                                               \
403 }
404
405 static FILE *msgcache_open_data_file(const gchar *file, guint version,
406                                      DataOpenMode mode,
407                                      gchar *buf, size_t buf_size)
408 {
409         FILE *fp;
410         gint32 data_ver;
411
412         cm_return_val_if_fail(file != NULL, NULL);
413
414         if (mode == DATA_WRITE) {
415                 int w_err = 0, wrote = 0;
416                 if ((fp = g_fopen(file, "wb")) == NULL) {
417                         FILE_OP_ERROR(file, "fopen");
418                         return NULL;
419                 }
420                 if (change_file_mode_rw(fp, file) < 0)
421                         FILE_OP_ERROR(file, "chmod");
422
423                 WRITE_CACHE_DATA_INT(version, fp);
424                 if (w_err != 0) {
425                         g_warning("failed to write int");
426                         fclose(fp);
427                         return NULL;
428                 }
429                 return fp;
430         }
431
432         /* check version */
433         if ((fp = g_fopen(file, "rb")) == NULL)
434                 debug_print("Mark/Cache file '%s' not found\n", file);
435         else {
436                 if (buf && buf_size > 0)
437                         setvbuf(fp, buf, _IOFBF, buf_size);
438                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
439                          version != bswap_32(data_ver)) {
440                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
441                                   file, bswap_32(data_ver), version);
442                         fclose(fp);
443                         fp = NULL;
444                 }
445                 data_ver = bswap_32(data_ver);
446         }
447         
448         if (mode == DATA_READ)
449                 return fp;
450
451         if (fp) {
452                 /* reopen with append mode */
453                 fclose(fp);
454                 if ((fp = g_fopen(file, "ab")) == NULL)
455                         FILE_OP_ERROR(file, "fopen");
456         } else {
457                 /* open with overwrite mode if mark file doesn't exist or
458                    version is different */
459                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
460                                             buf_size);
461         }
462
463         return fp;
464 }
465
466 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
467                                          StringConverter *conv)
468 {
469         gchar *tmpstr = NULL;
470         size_t ni;
471         guint32 len;
472
473         *str = NULL;
474         if (!swapping) {
475                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
476                     len > G_MAXINT) {
477                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
478                                   "of %zd bytes at offset %ld", ni, sizeof(len),
479                                   ftell(fp));
480                         return -1;
481                 }
482         } else {
483                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
484                     bswap_32(len) > G_MAXINT) {
485                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
486                                   "of %zd bytes at offset %ld", ni, sizeof(len),
487                                   ftell(fp));
488                         return -1;
489                 }
490                 len = bswap_32(len);
491         }
492
493         if (len == 0)
494                 return 0;
495
496         tmpstr = g_try_malloc(len + 1);
497
498         if(tmpstr == NULL) {
499                 return -1;
500         }
501
502         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
503                 g_warning("read_data_str: Cache data corrupted, read %zd of %u "
504                           "bytes at offset %ld",
505                           ni, len, ftell(fp));
506                 g_free(tmpstr);
507                 return -1;
508         }
509         tmpstr[len] = 0;
510
511         if (conv != NULL) {
512                 *str = conv->convert(conv, tmpstr);
513                 g_free(tmpstr);
514         } else 
515                 *str = tmpstr;
516
517         return len;
518 }
519
520 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
521                                          StringConverter *conv)
522 {
523         gchar *tmpstr = NULL;
524
525         *str = NULL;
526
527         if (len == 0)
528                 return 0;
529
530         if(len > 2*1024*1024) {
531                 g_warning("read_data_str: refusing to allocate %d bytes.", len);
532                 return -1;
533         }
534
535         tmpstr = g_try_malloc(len + 1);
536
537         if(tmpstr == NULL) {
538                 return -1;
539         }
540
541         memcpy(tmpstr, src, len);
542         tmpstr[len] = 0;
543
544         if (conv != NULL) {
545                 *str = conv->convert(conv, tmpstr);
546                 g_free(tmpstr);
547         } else 
548                 *str = tmpstr;
549
550         return len;
551 }
552
553 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
554 {
555         CharsetConverter *charsetconv = (CharsetConverter *) conv;
556
557         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
558 }
559
560 static void strconv_charset_free(StringConverter *conv)
561 {
562         CharsetConverter *charsetconv = (CharsetConverter *) conv;
563
564         g_free(charsetconv->srccharset);
565         g_free(charsetconv->dstcharset);
566 }
567
568 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
569 {
570         MsgCache *cache;
571         FILE *fp;
572         MsgInfo *msginfo;
573         MsgTmpFlags tmp_flags = 0;
574         gchar file_buf[BUFFSIZE];
575         guint32 num;
576         guint refnum;
577         gboolean error = FALSE;
578         StringConverter *conv = NULL;
579         gchar *srccharset = NULL;
580         const gchar *dstcharset = NULL;
581         gchar *ref = NULL;
582         guint memusage = 0;
583         gint tmp_len = 0, map_len = -1;
584         char *cache_data = NULL;
585         struct stat st;
586
587         cm_return_val_if_fail(cache_file != NULL, NULL);
588         cm_return_val_if_fail(item != NULL, NULL);
589
590         swapping = TRUE;
591
592         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
593          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
594          * it means it's the old version (not little-endian) on a big-endian machine. The code has
595          * no effect on x86 as their file doesn't change. */
596
597         if ((fp = msgcache_open_data_file
598                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
599                 if ((fp = msgcache_open_data_file
600                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
601                         return NULL;
602                 else
603                         swapping = FALSE;
604         }
605
606         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
607
608         if (folder_has_parent_of_type(item, F_QUEUE)) {
609                 tmp_flags |= MSG_QUEUED;
610         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
611                 tmp_flags |= MSG_DRAFT;
612         }
613
614         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0) {
615                 fclose(fp);
616                 return NULL;
617         }
618         dstcharset = CS_UTF_8;
619         if (srccharset == NULL || dstcharset == NULL) {
620                 conv = NULL;
621         } else if (strcmp(srccharset, dstcharset) == 0) {
622                 debug_print("using Noop Converter\n");
623
624                 conv = NULL;
625         } else {
626                 CharsetConverter *charsetconv;
627
628                 debug_print("using CharsetConverter\n");
629
630                 charsetconv = g_new0(CharsetConverter, 1);
631                 charsetconv->converter.convert = strconv_charset_convert;
632                 charsetconv->converter.free = strconv_charset_free;
633                 charsetconv->srccharset = g_strdup(srccharset);
634                 charsetconv->dstcharset = g_strdup(dstcharset);
635
636                 conv = (StringConverter *) charsetconv;
637         }
638         g_free(srccharset);
639
640         cache = msgcache_new();
641
642         if (msgcache_use_mmap_read == TRUE) {
643                 if (fstat(fileno(fp), &st) >= 0)
644                         map_len = st.st_size;
645                 else
646                         map_len = -1;
647                 if (map_len > 0) {
648 #ifdef G_OS_WIN32
649                         cache_data = NULL;
650                         HANDLE hFile, hMapping;
651                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
652                         if (hFile == (HANDLE) -1)
653                                 goto w32_fail;
654                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
655                         if (!hMapping)
656                                 goto w32_fail;
657                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
658                         CloseHandle (hMapping);
659                 w32_fail:
660                         ;
661 #else
662                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
663 #endif
664                 }
665         } else {
666                 cache_data = NULL;
667         }
668         if (cache_data != NULL && cache_data != MAP_FAILED) {
669                 int rem_len = map_len-ftell(fp);
670                 char *walk_data = cache_data+ftell(fp);
671
672                 while(rem_len > 0) {
673                         GET_CACHE_DATA_INT(num);
674                         
675                         msginfo = procmsg_msginfo_new();
676                         msginfo->msgnum = num;
677                         memusage += sizeof(MsgInfo);
678
679                         GET_CACHE_DATA_INT(msginfo->size);
680                         GET_CACHE_DATA_INT(msginfo->mtime);
681                         GET_CACHE_DATA_INT(msginfo->date_t);
682                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
683
684                         GET_CACHE_DATA(msginfo->fromname, memusage);
685
686                         GET_CACHE_DATA(msginfo->date, memusage);
687                         GET_CACHE_DATA(msginfo->from, memusage);
688                         GET_CACHE_DATA(msginfo->to, memusage);
689                         GET_CACHE_DATA(msginfo->cc, memusage);
690                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
691                         GET_CACHE_DATA(msginfo->subject, memusage);
692                         GET_CACHE_DATA(msginfo->msgid, memusage);
693                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
694                         GET_CACHE_DATA(msginfo->xref, memusage);
695
696                         GET_CACHE_DATA_INT(msginfo->planned_download);
697                         GET_CACHE_DATA_INT(msginfo->total_size);
698                         GET_CACHE_DATA_INT(refnum);
699
700                         for (; refnum != 0; refnum--) {
701                                 ref = NULL;
702
703                                 GET_CACHE_DATA(ref, memusage);
704
705                                 if (ref && *ref)
706                                         msginfo->references =
707                                                 g_slist_prepend(msginfo->references, ref);
708                         }
709                         if (msginfo->references)
710                                 msginfo->references =
711                                         g_slist_reverse(msginfo->references);
712
713                         msginfo->folder = item;
714                         msginfo->flags.tmp_flags |= tmp_flags;
715
716                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
717                         if(msginfo->msgid)
718                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
719                 }
720         } else {
721                 while (fread(&num, sizeof(num), 1, fp) == 1) {
722                         if (swapping)
723                                 num = bswap_32(num);
724
725                         msginfo = procmsg_msginfo_new();
726                         msginfo->msgnum = num;
727                         memusage += sizeof(MsgInfo);
728
729                         READ_CACHE_DATA_INT(msginfo->size, fp);
730                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
731                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
732                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
733
734                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
735
736                         READ_CACHE_DATA(msginfo->date, fp, memusage);
737                         READ_CACHE_DATA(msginfo->from, fp, memusage);
738                         READ_CACHE_DATA(msginfo->to, fp, memusage);
739                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
740                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
741                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
742                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
743                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
744                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
745
746                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
747                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
748                         READ_CACHE_DATA_INT(refnum, fp);
749
750                         for (; refnum != 0; refnum--) {
751                                 ref = NULL;
752
753                                 READ_CACHE_DATA(ref, fp, memusage);
754
755                                 if (ref && *ref)
756                                         msginfo->references =
757                                                 g_slist_prepend(msginfo->references, ref);
758                         }
759                         if (msginfo->references)
760                                 msginfo->references =
761                                         g_slist_reverse(msginfo->references);
762
763                         msginfo->folder = item;
764                         msginfo->flags.tmp_flags |= tmp_flags;
765
766                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
767                         if(msginfo->msgid)
768                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
769                 }
770         }
771 bail_err:
772         if (cache_data != NULL && cache_data != MAP_FAILED) {
773 #ifdef G_OS_WIN32
774                 UnmapViewOfFile((void*) cache_data);
775 #else
776                 munmap(cache_data, map_len);
777 #endif
778         }
779         fclose(fp);
780         if (conv != NULL) {
781                 if (conv->free != NULL)
782                         conv->free(conv);
783                 g_free(conv);
784         }
785
786         if(error) {
787                 msgcache_destroy(cache);
788                 return NULL;
789         }
790
791         cache->last_access = time(NULL);
792         cache->memusage = memusage;
793
794         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
795         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
796
797         return cache;
798 }
799
800 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
801 {
802         FILE *fp;
803         MsgInfo *msginfo;
804         MsgPermFlags perm_flags;
805         guint32 num;
806         gint map_len = -1;
807         char *cache_data = NULL;
808         struct stat st;
809         gboolean error = FALSE;
810
811         swapping = TRUE;
812
813         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
814          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
815          * it means it's the old version (not little-endian) on a big-endian machine. The code has
816          * no effect on x86 as their file doesn't change. */
817
818         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
819                 /* see if it isn't swapped ? */
820                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
821                         return;
822                 else
823                         swapping = FALSE; /* yay */
824         }
825         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
826         
827         if (msgcache_use_mmap_read) {
828                 if (fstat(fileno(fp), &st) >= 0)
829                         map_len = st.st_size;
830                 else
831                         map_len = -1;
832                 if (map_len > 0) {
833 #ifdef G_OS_WIN32
834                         cache_data = NULL;
835                         HANDLE hFile, hMapping;
836                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
837                         if (hFile == (HANDLE) -1)
838                                 goto w32_fail2;
839                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
840                         if (!hMapping)
841                                 goto w32_fail2;
842                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
843                         CloseHandle (hMapping);
844                 w32_fail2:
845                         ;
846 #else
847                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
848 #endif
849                 }
850         } else {
851                 cache_data = NULL;
852         }
853         if (cache_data != NULL && cache_data != MAP_FAILED) {
854                 int rem_len = map_len-ftell(fp);
855                 char *walk_data = cache_data+ftell(fp);
856
857                 while(rem_len > 0) {
858                         GET_CACHE_DATA_INT(num);
859                         GET_CACHE_DATA_INT(perm_flags);
860                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
861                         if(msginfo) {
862                                 msginfo->flags.perm_flags = perm_flags;
863                         }
864                 }
865         } else {
866                 while (fread(&num, sizeof(num), 1, fp) == 1) {
867                         if (swapping)
868                                 num = bswap_32(num);
869                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) {
870                                 error = TRUE;
871                                 break;
872                         }
873                         if (swapping)
874                                 perm_flags = bswap_32(perm_flags);
875                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
876                         if(msginfo) {
877                                 msginfo->flags.perm_flags = perm_flags;
878                         }
879                 }       
880         }
881 bail_err:
882         if (cache_data != NULL && cache_data != MAP_FAILED) {
883 #ifdef G_OS_WIN32
884                 UnmapViewOfFile((void*) cache_data);
885 #else
886                 munmap(cache_data, map_len);
887 #endif
888         }
889         fclose(fp);
890         if (error) {
891                 debug_print("error reading cache mark from %s\n", mark_file);
892         }
893 }
894
895 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
896 {
897         FILE *fp;
898         MsgInfo *msginfo;
899         guint32 num;
900         gint map_len = -1;
901         char *cache_data = NULL;
902         struct stat st;
903         gboolean error = FALSE;
904
905         swapping = TRUE;
906
907         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
908          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
909          * it means it's the old version (not little-endian) on a big-endian machine. The code has
910          * no effect on x86 as their file doesn't change. */
911
912         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
913                 /* see if it isn't swapped ? */
914                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
915                         return;
916                 else
917                         swapping = FALSE; /* yay */
918         }
919         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
920         
921         if (msgcache_use_mmap_read) {
922                 if (fstat(fileno(fp), &st) >= 0)
923                         map_len = st.st_size;
924                 else
925                         map_len = -1;
926                 if (map_len > 0) {
927 #ifdef G_OS_WIN32
928                         cache_data = NULL;
929                         HANDLE hFile, hMapping;
930                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
931                         if (hFile == (HANDLE) -1)
932                                 goto w32_fail6;
933                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
934                         if (!hMapping)
935                                 goto w32_fail6;
936                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
937                         CloseHandle (hMapping);
938                 w32_fail6:
939                         ;
940 #else
941                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
942 #endif
943                 }
944         } else {
945                 cache_data = NULL;
946         }
947         if (cache_data != NULL && cache_data != MAP_FAILED) {
948                 int rem_len = map_len-ftell(fp);
949                 char *walk_data = cache_data+ftell(fp);
950
951                 while(rem_len > 0) {
952                         gint id = -1;
953                         GET_CACHE_DATA_INT(num);
954                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
955                         if(msginfo) {
956                                 g_slist_free(msginfo->tags);
957                                 msginfo->tags = NULL;
958                                 do {
959                                         GET_CACHE_DATA_INT(id);
960                                         if (id > 0) {
961                                                 msginfo->tags = g_slist_prepend(
962                                                         msginfo->tags, 
963                                                         GINT_TO_POINTER(id));
964                                         }
965                                 } while (id > 0);
966                                 msginfo->tags = g_slist_reverse(msginfo->tags);
967                         }
968                 }
969         } else {
970                 while (fread(&num, sizeof(num), 1, fp) == 1) {
971                         gint id = -1;
972                         if (swapping)
973                                 num = bswap_32(num);
974                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
975                         if(msginfo) {
976                                 g_slist_free(msginfo->tags);
977                                 msginfo->tags = NULL;
978                                 do {
979                                         if (fread(&id, sizeof(id), 1, fp) != 1) 
980                                                 id = -1;
981                                         if (swapping)
982                                                 id = bswap_32(id);
983                                         if (id > 0) {
984                                                 msginfo->tags = g_slist_prepend(
985                                                         msginfo->tags, 
986                                                         GINT_TO_POINTER(id));
987                                         }
988                                 } while (id > 0);
989                                 msginfo->tags = g_slist_reverse(msginfo->tags);
990                         }
991                 }
992         }
993 bail_err:
994         if (cache_data != NULL && cache_data != MAP_FAILED) {
995 #ifdef G_OS_WIN32
996                 UnmapViewOfFile((void*) cache_data);
997 #else
998                 munmap(cache_data, map_len);
999 #endif
1000         }
1001         fclose(fp);
1002         if (error) {
1003                 debug_print("error reading cache tags from %s\n", tags_file);
1004         }
1005 }
1006
1007 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
1008 {
1009         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
1010         GSList *cur;
1011         int w_err = 0, wrote = 0;
1012
1013         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1014         WRITE_CACHE_DATA_INT(msginfo->size, fp);
1015         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
1016         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
1017         WRITE_CACHE_DATA_INT(flags, fp);
1018
1019         WRITE_CACHE_DATA(msginfo->fromname, fp);
1020
1021         WRITE_CACHE_DATA(msginfo->date, fp);
1022         WRITE_CACHE_DATA(msginfo->from, fp);
1023         WRITE_CACHE_DATA(msginfo->to, fp);
1024         WRITE_CACHE_DATA(msginfo->cc, fp);
1025         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
1026         WRITE_CACHE_DATA(msginfo->subject, fp);
1027         WRITE_CACHE_DATA(msginfo->msgid, fp);
1028         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
1029         WRITE_CACHE_DATA(msginfo->xref, fp);
1030         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
1031         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
1032         
1033         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
1034
1035         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
1036                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
1037         }
1038         return w_err ? -1 : wrote;
1039 }
1040
1041 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
1042 {
1043         MsgPermFlags flags = msginfo->flags.perm_flags;
1044         int w_err = 0, wrote = 0;
1045         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1046         WRITE_CACHE_DATA_INT(flags, fp);
1047         return w_err ? -1 : wrote;
1048 }
1049
1050 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
1051 {
1052         GSList *cur = msginfo->tags;
1053         int w_err = 0, wrote = 0;
1054
1055         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1056         for (; cur; cur = cur->next) {
1057                 gint id = GPOINTER_TO_INT(cur->data);
1058                 if (tags_get_tag(id) != NULL) {
1059                         WRITE_CACHE_DATA_INT(id, fp);
1060                 }
1061         }
1062         WRITE_CACHE_DATA_INT(-1, fp);
1063
1064         return w_err ? -1 : wrote;
1065 }
1066
1067 struct write_fps
1068 {
1069         FILE *cache_fp;
1070         FILE *mark_fp;
1071         FILE *tags_fp;
1072         int error;
1073         guint cache_size;
1074         guint mark_size;
1075         guint tags_size;
1076 };
1077
1078 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1079 {
1080         MsgInfo *msginfo;
1081         struct write_fps *write_fps;
1082         int tmp;
1083
1084         msginfo = (MsgInfo *)value;
1085         write_fps = user_data;
1086
1087         if (write_fps->cache_fp) {
1088                 tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1089                 if (tmp < 0)
1090                         write_fps->error = 1;
1091                 else
1092                         write_fps->cache_size += tmp;
1093         }
1094         if (write_fps->mark_fp) {
1095         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1096                 if (tmp < 0)
1097                         write_fps->error = 1;
1098                 else
1099                         write_fps->mark_size += tmp;
1100                 }
1101         if (write_fps->tags_fp) {
1102                 tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1103                 if (tmp < 0)
1104                         write_fps->error = 1;
1105                 else
1106                         write_fps->tags_size += tmp;
1107         }
1108 }
1109
1110 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1111 {
1112         struct write_fps write_fps;
1113         gchar *new_cache, *new_mark, *new_tags;
1114         int w_err = 0, wrote = 0;
1115
1116         START_TIMING("");
1117         cm_return_val_if_fail(cache != NULL, -1);
1118
1119         new_cache = g_strconcat(cache_file, ".new", NULL);
1120         new_mark  = g_strconcat(mark_file, ".new", NULL);
1121         new_tags  = g_strconcat(tags_file, ".new", NULL);
1122
1123         write_fps.error = 0;
1124         write_fps.cache_size = 0;
1125         write_fps.mark_size = 0;
1126         write_fps.tags_size = 0;
1127
1128         /* open files and write headers */
1129
1130         if (cache_file) {
1131                 write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1132                         DATA_WRITE, NULL, 0);
1133                 if (write_fps.cache_fp == NULL) {
1134                         g_free(new_cache);
1135                         g_free(new_mark);
1136                         g_free(new_tags);
1137                         return -1;
1138                 }
1139                 WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1140         } else {
1141                 write_fps.cache_fp = NULL;
1142         }
1143
1144         if (w_err != 0) {
1145                 g_warning("failed to write charset");
1146                 if (write_fps.cache_fp)
1147                         fclose(write_fps.cache_fp);
1148                 claws_unlink(new_cache);
1149                 g_free(new_cache);
1150                 g_free(new_mark);
1151                 g_free(new_tags);
1152                 return -1;
1153         }
1154
1155         if (mark_file) {
1156                 write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1157                         DATA_WRITE, NULL, 0);
1158                 if (write_fps.mark_fp == NULL) {
1159                         if (write_fps.cache_fp)
1160                                 fclose(write_fps.cache_fp);
1161                         claws_unlink(new_cache);
1162                         g_free(new_cache);
1163                         g_free(new_mark);
1164                         g_free(new_tags);
1165                         return -1;
1166                 }
1167         } else {
1168                 write_fps.mark_fp = NULL;
1169         }
1170
1171         if (tags_file) {
1172                 write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1173                         DATA_WRITE, NULL, 0);
1174                 if (write_fps.tags_fp == NULL) {
1175                         if (write_fps.cache_fp)
1176                                 fclose(write_fps.cache_fp);
1177                         if (write_fps.mark_fp)
1178                                 fclose(write_fps.mark_fp);
1179                         claws_unlink(new_cache);
1180                         claws_unlink(new_mark);
1181                         g_free(new_cache);
1182                         g_free(new_mark);
1183                         g_free(new_tags);
1184                         return -1;
1185                 }
1186         } else {
1187                 write_fps.tags_fp = NULL;
1188         }
1189
1190         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1191
1192         if (write_fps.cache_fp && change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1193                 FILE_OP_ERROR(new_cache, "chmod");
1194
1195         /* headers written, note file size */
1196         if (write_fps.cache_fp)
1197                 write_fps.cache_size = ftell(write_fps.cache_fp);
1198         if (write_fps.mark_fp)
1199                 write_fps.mark_size = ftell(write_fps.mark_fp);
1200         if (write_fps.tags_fp)
1201                 write_fps.tags_size = ftell(write_fps.tags_fp);
1202
1203 #ifdef HAVE_FWRITE_UNLOCKED
1204         /* lock files for write once (instead of once per fwrite) */
1205         if (write_fps.cache_fp)
1206                 flockfile(write_fps.cache_fp);
1207         if (write_fps.mark_fp)
1208                 flockfile(write_fps.mark_fp);
1209         if (write_fps.tags_fp)
1210                 flockfile(write_fps.tags_fp);
1211 #endif
1212         /* write data to the files */
1213         g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1214 #ifdef HAVE_FWRITE_UNLOCKED
1215         /* unlock files */
1216         if (write_fps.cache_fp)
1217                 funlockfile(write_fps.cache_fp);
1218         if (write_fps.mark_fp)
1219                 funlockfile(write_fps.mark_fp);
1220         if (write_fps.tags_fp)
1221                 funlockfile(write_fps.tags_fp);
1222 #endif
1223
1224         /* close files */
1225         if (write_fps.cache_fp)
1226                 write_fps.error |= (safe_fclose(write_fps.cache_fp) != 0);
1227         if (write_fps.mark_fp)
1228                 write_fps.error |= (safe_fclose(write_fps.mark_fp) != 0);
1229         if (write_fps.tags_fp)
1230                 write_fps.error |= (safe_fclose(write_fps.tags_fp) != 0);
1231
1232
1233         if (write_fps.error != 0) {
1234                 /* in case of error, forget all */
1235                 claws_unlink(new_cache);
1236                 claws_unlink(new_mark);
1237                 claws_unlink(new_tags);
1238                 g_free(new_cache);
1239                 g_free(new_mark);
1240                 g_free(new_tags);
1241                 return -1;
1242         } else {
1243                 /* switch files */
1244                 if (cache_file)
1245                         move_file(new_cache, cache_file, TRUE);
1246                 if (mark_file)
1247                         move_file(new_mark, mark_file, TRUE);
1248                 if (tags_file)
1249                         move_file(new_tags, tags_file, TRUE);
1250                 cache->last_access = time(NULL);
1251         }
1252
1253         g_free(new_cache);
1254         g_free(new_mark);
1255         g_free(new_tags);
1256         debug_print("done.\n");
1257         END_TIMING();
1258         return 0;
1259 }
1260