2012-07-03 [mir] 3.8.1cvs5
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2012 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  * 
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #endif
23
24 #include "defs.h"
25
26 #define _GNU_SOURCE
27 #include <stdio.h>
28
29 #include <glib.h>
30 #include <glib/gi18n.h>
31 #ifdef _WIN32
32 # include <w32lib.h>
33 # define MAP_FAILED     ((char *) -1)
34 #else
35 # include <sys/mman.h>
36 #endif
37 #include <sys/types.h>
38 #include <sys/stat.h>
39
40 #include <time.h>
41
42 #include "msgcache.h"
43 #include "utils.h"
44 #include "procmsg.h"
45 #include "codeconv.h"
46 #include "timing.h"
47 #include "tags.h"
48 #include "prefs_common.h"
49
50 #ifdef HAVE_FWRITE_UNLOCKED
51 #define SC_FWRITE fwrite_unlocked
52 #else
53 #define SC_FWRITE fwrite
54 #endif
55
56 #if G_BYTE_ORDER == G_BIG_ENDIAN
57 #define bswap_32(x) \
58      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
59       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
60      
61 #define MMAP_TO_GUINT32(x)      \
62         (((x[3]&0xff)) |        \
63          ((x[2]&0xff) << 8) |   \
64          ((x[1]&0xff) << 16) |  \
65          ((x[0]&0xff) << 24))
66
67 #define MMAP_TO_GUINT32_SWAPPED(x)      \
68         (((x[0]&0xff)) |                \
69          ((x[1]&0xff) << 8) |           \
70          ((x[2]&0xff) << 16) |          \
71          ((x[3]&0xff) << 24))
72
73 static gboolean msgcache_use_mmap_read = TRUE;
74
75 #else
76 #define bswap_32(x) (x)
77
78 #define MMAP_TO_GUINT32(x)      \
79         (((x[0]&0xff)) |        \
80          ((x[1]&0xff) << 8) |   \
81          ((x[2]&0xff) << 16) |  \
82          ((x[3]&0xff) << 24))
83
84 #define MMAP_TO_GUINT32_SWAPPED(x)      \
85         (((x[0]&0xff)) |                \
86          ((x[1]&0xff) << 8) |           \
87          ((x[2]&0xff) << 16) |          \
88          ((x[3]&0xff) << 24))
89
90 static gboolean msgcache_use_mmap_read = TRUE;
91 #endif
92
93 static gboolean swapping = TRUE;
94
95 typedef enum
96 {
97         DATA_READ,
98         DATA_WRITE,
99         DATA_APPEND
100 } DataOpenMode;
101
102 struct _MsgCache {
103         GHashTable      *msgnum_table;
104         GHashTable      *msgid_table;
105         guint            memusage;
106         time_t           last_access;
107 };
108
109 typedef struct _StringConverter StringConverter;
110 struct _StringConverter {
111         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
112         void   (*free)    (StringConverter *converter);
113 };
114
115 typedef struct _StrdupConverter StrdupConverter;
116 struct _StrdupConverter {
117         StringConverter converter;
118 };
119
120 typedef struct _CharsetConverter CharsetConverter;
121 struct _CharsetConverter {
122         StringConverter converter;
123
124         gchar *srccharset;
125         gchar *dstcharset;
126 };
127
128 MsgCache *msgcache_new(void)
129 {
130         MsgCache *cache;
131         
132         cache = g_new0(MsgCache, 1),
133         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
134         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
135         cache->last_access = time(NULL);
136
137         return cache;
138 }
139
140 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
141 {
142         procmsg_msginfo_free((MsgInfo *)msginfo);
143         return TRUE;
144 }                                                                                         
145
146 void msgcache_destroy(MsgCache *cache)
147 {
148         cm_return_if_fail(cache != NULL);
149
150         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
151         g_hash_table_destroy(cache->msgid_table);
152         g_hash_table_destroy(cache->msgnum_table);
153         g_free(cache);
154 }
155
156 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
157 {
158         MsgInfo *newmsginfo;
159
160         cm_return_if_fail(cache != NULL);
161         cm_return_if_fail(msginfo != NULL);
162
163         newmsginfo = procmsg_msginfo_new_ref(msginfo);
164         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
165         if(newmsginfo->msgid != NULL)
166                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
167         cache->memusage += procmsg_msginfo_memusage(msginfo);
168         cache->last_access = time(NULL);
169
170         msginfo->folder->cache_dirty = TRUE;
171
172         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
173 }
174
175 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
176 {
177         MsgInfo *msginfo;
178
179         cm_return_if_fail(cache != NULL);
180
181         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
182         if(!msginfo)
183                 return;
184
185         cache->memusage -= procmsg_msginfo_memusage(msginfo);
186         if(msginfo->msgid)
187                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
188         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
189         procmsg_msginfo_free(msginfo);
190         cache->last_access = time(NULL);
191
192         msginfo->folder->cache_dirty = TRUE;
193
194         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
195 }
196
197 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
198 {
199         MsgInfo *oldmsginfo, *newmsginfo;
200         
201         cm_return_if_fail(cache != NULL);
202         cm_return_if_fail(msginfo != NULL);
203
204         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
205         if(oldmsginfo && oldmsginfo->msgid) 
206                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
207         if (oldmsginfo) {
208                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
209                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
210                 procmsg_msginfo_free(oldmsginfo);
211         }
212
213         newmsginfo = procmsg_msginfo_new_ref(msginfo);
214         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
215         if(newmsginfo->msgid)
216                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
217         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
218         cache->last_access = time(NULL);
219         
220         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
221
222         msginfo->folder->cache_dirty = TRUE;
223
224         return;
225 }
226
227 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
228 {
229         MsgInfo *msginfo;
230
231         cm_return_val_if_fail(cache != NULL, NULL);
232
233         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
234         if(!msginfo)
235                 return NULL;
236         cache->last_access = time(NULL);
237         
238         return procmsg_msginfo_new_ref(msginfo);
239 }
240
241 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
242 {
243         MsgInfo *msginfo;
244         
245         cm_return_val_if_fail(cache != NULL, NULL);
246         cm_return_val_if_fail(msgid != NULL, NULL);
247
248         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
249         if(!msginfo)
250                 return NULL;
251         cache->last_access = time(NULL);
252         
253         return procmsg_msginfo_new_ref(msginfo);        
254 }
255
256 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
257 {
258         MsgInfoList **listptr = user_data;
259         MsgInfo *msginfo = value;
260
261         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
262 }
263
264 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
265 {
266         MsgInfoList *msg_list = NULL;
267         START_TIMING("");
268         cm_return_val_if_fail(cache != NULL, NULL);
269
270         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
271         cache->last_access = time(NULL);
272         
273         msg_list = g_slist_reverse(msg_list);
274         END_TIMING();
275         return msg_list;
276 }
277
278 time_t msgcache_get_last_access_time(MsgCache *cache)
279 {
280         cm_return_val_if_fail(cache != NULL, 0);
281         
282         return cache->last_access;
283 }
284
285 gint msgcache_get_memory_usage(MsgCache *cache)
286 {
287         cm_return_val_if_fail(cache != NULL, 0);
288
289         return cache->memusage;
290 }
291
292 /*
293  *  Cache saving functions
294  */
295
296 #define READ_CACHE_DATA(data, fp, total_len) \
297 { \
298         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
299                 procmsg_msginfo_free(msginfo); \
300                 error = TRUE; \
301                 goto bail_err; \
302         } \
303         total_len += tmp_len; \
304 }
305
306 #define READ_CACHE_DATA_INT(n, fp) \
307 { \
308         guint32 idata; \
309         size_t ni; \
310  \
311         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
312                 g_warning("read_int: Cache data corrupted, read %zd of %zd at " \
313                           "offset %ld\n", ni, sizeof(idata), ftell(fp)); \
314                 procmsg_msginfo_free(msginfo); \
315                 error = TRUE; \
316                 goto bail_err; \
317         } else \
318                 n = swapping ? bswap_32(idata) : (idata);\
319 }
320
321 #define GET_CACHE_DATA_INT(n)                                                                   \
322 {                                                                                               \
323         if (rem_len < 4) {                                                                      \
324                 g_print("error at rem_len:%d\n", rem_len);                                      \
325                 error = TRUE;                                                                   \
326                 goto bail_err;                                                                  \
327         }                                                                                       \
328         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data)));     \
329         walk_data += 4; rem_len -= 4;                                                           \
330 }
331
332 #define GET_CACHE_DATA(data, total_len) \
333 { \
334         GET_CACHE_DATA_INT(tmp_len);    \
335         if (rem_len < tmp_len) {                                                                \
336                 g_print("error at rem_len:%d (tmp_len %d)\n", rem_len, tmp_len);                \
337                 error = TRUE;                                                                   \
338                 goto bail_err;                                                                  \
339         }                                                                                       \
340         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
341                 g_print("error at rem_len:%d\n", rem_len);\
342                 procmsg_msginfo_free(msginfo); \
343                 error = TRUE; \
344                 goto bail_err; \
345         } \
346         total_len += tmp_len; \
347         walk_data += tmp_len; rem_len -= tmp_len; \
348 }
349
350
351 #define WRITE_CACHE_DATA_INT(n, fp)                     \
352 {                                                       \
353         guint32 idata;                                  \
354                                                         \
355         idata = (guint32)bswap_32(n);                   \
356         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
357                 w_err = 1;                              \
358         wrote += 4;                                     \
359 }
360
361 #define PUT_CACHE_DATA_INT(n)                           \
362 {                                                       \
363         walk_data[0]=(((guint32)n)&0x000000ff);                 \
364         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
365         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
366         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
367         walk_data += 4;                                 \
368         wrote += 4;                                     \
369 }
370
371 #define WRITE_CACHE_DATA(data, fp) \
372 { \
373         size_t len;                                     \
374         if (data == NULL)                               \
375                 len = 0;                                \
376         else                                            \
377                 len = strlen(data);                     \
378         WRITE_CACHE_DATA_INT(len, fp);                  \
379         if (w_err == 0 && len > 0) {                    \
380                 if (SC_FWRITE(data, 1, len, fp) != len) \
381                         w_err = 1;                      \
382                 wrote += len;                           \
383         } \
384 }
385
386 #define PUT_CACHE_DATA(data)                            \
387 {                                                       \
388         size_t len;                                     \
389         if (data == NULL)                               \
390                 len = 0;                                \
391         else                                            \
392                 len = strlen(data);                     \
393         PUT_CACHE_DATA_INT(len);                        \
394         if (len > 0) {                                  \
395                 memcpy(walk_data, data, len);           \
396                 walk_data += len;                       \
397                 wrote += len;                           \
398         }                                               \
399 }
400
401 static FILE *msgcache_open_data_file(const gchar *file, guint version,
402                                      DataOpenMode mode,
403                                      gchar *buf, size_t buf_size)
404 {
405         FILE *fp;
406         gint32 data_ver;
407
408         cm_return_val_if_fail(file != NULL, NULL);
409
410         if (mode == DATA_WRITE) {
411                 int w_err = 0, wrote = 0;
412                 if ((fp = g_fopen(file, "wb")) == NULL) {
413                         FILE_OP_ERROR(file, "fopen");
414                         return NULL;
415                 }
416                 if (change_file_mode_rw(fp, file) < 0)
417                         FILE_OP_ERROR(file, "chmod");
418
419                 WRITE_CACHE_DATA_INT(version, fp);
420                 if (w_err != 0) {
421                         g_warning("failed to write int\n");
422                         fclose(fp);
423                         return NULL;
424                 }
425                 return fp;
426         }
427
428         /* check version */
429         if ((fp = g_fopen(file, "rb")) == NULL)
430                 debug_print("Mark/Cache file '%s' not found\n", file);
431         else {
432                 if (buf && buf_size > 0)
433                         setvbuf(fp, buf, _IOFBF, buf_size);
434                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
435                          version != bswap_32(data_ver)) {
436                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
437                                   file, bswap_32(data_ver), version);
438                         fclose(fp);
439                         fp = NULL;
440                 }
441                 data_ver = bswap_32(data_ver);
442         }
443         
444         if (mode == DATA_READ)
445                 return fp;
446
447         if (fp) {
448                 /* reopen with append mode */
449                 fclose(fp);
450                 if ((fp = g_fopen(file, "ab")) == NULL)
451                         FILE_OP_ERROR(file, "fopen");
452         } else {
453                 /* open with overwrite mode if mark file doesn't exist or
454                    version is different */
455                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
456                                             buf_size);
457         }
458
459         return fp;
460 }
461
462 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
463                                          StringConverter *conv)
464 {
465         gchar *tmpstr = NULL;
466         size_t ni;
467         guint32 len;
468
469         *str = NULL;
470         if (!swapping) {
471                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
472                     len > G_MAXINT) {
473                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
474                                   "of %zd bytes at offset %ld\n", ni, sizeof(len), 
475                                   ftell(fp));
476                         return -1;
477                 }
478         } else {
479                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
480                     bswap_32(len) > G_MAXINT) {
481                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
482                                   "of %zd bytes at offset %ld\n", ni, sizeof(len), 
483                                   ftell(fp));
484                         return -1;
485                 }
486                 len = bswap_32(len);
487         }
488
489         if (len == 0)
490                 return 0;
491
492         tmpstr = g_try_malloc(len + 1);
493
494         if(tmpstr == NULL) {
495                 return -1;
496         }
497
498         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
499                 g_warning("read_data_str: Cache data corrupted, read %zd of %u "
500                           "bytes at offset %ld\n", 
501                           ni, len, ftell(fp));
502                 g_free(tmpstr);
503                 return -1;
504         }
505         tmpstr[len] = 0;
506
507         if (conv != NULL) {
508                 *str = conv->convert(conv, tmpstr);
509                 g_free(tmpstr);
510         } else 
511                 *str = tmpstr;
512
513         return len;
514 }
515
516 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
517                                          StringConverter *conv)
518 {
519         gchar *tmpstr = NULL;
520
521         *str = NULL;
522
523         if (len == 0)
524                 return 0;
525
526         if(len > 2*1024*1024) {
527                 g_warning("read_data_str: refusing to allocate %d bytes.\n", len);
528                 return -1;
529         }
530
531         tmpstr = g_try_malloc(len + 1);
532
533         if(tmpstr == NULL) {
534                 return -1;
535         }
536
537         memcpy(tmpstr, src, len);
538         tmpstr[len] = 0;
539
540         if (conv != NULL) {
541                 *str = conv->convert(conv, tmpstr);
542                 g_free(tmpstr);
543         } else 
544                 *str = tmpstr;
545
546         return len;
547 }
548
549 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
550 {
551         CharsetConverter *charsetconv = (CharsetConverter *) conv;
552
553         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
554 }
555
556 static void strconv_charset_free(StringConverter *conv)
557 {
558         CharsetConverter *charsetconv = (CharsetConverter *) conv;
559
560         g_free(charsetconv->srccharset);
561         g_free(charsetconv->dstcharset);
562 }
563
564 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
565 {
566         MsgCache *cache;
567         FILE *fp;
568         MsgInfo *msginfo;
569         MsgTmpFlags tmp_flags = 0;
570         gchar file_buf[BUFFSIZE];
571         guint32 num;
572         guint refnum;
573         gboolean error = FALSE;
574         StringConverter *conv = NULL;
575         gchar *srccharset = NULL;
576         const gchar *dstcharset = NULL;
577         gchar *ref = NULL;
578         guint memusage = 0;
579         gint tmp_len = 0, map_len = -1;
580         char *cache_data = NULL;
581         struct stat st;
582
583         cm_return_val_if_fail(cache_file != NULL, NULL);
584         cm_return_val_if_fail(item != NULL, NULL);
585
586         swapping = TRUE;
587
588         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
589          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
590          * it means it's the old version (not little-endian) on a big-endian machine. The code has
591          * no effect on x86 as their file doesn't change. */
592
593         if ((fp = msgcache_open_data_file
594                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
595                 if ((fp = msgcache_open_data_file
596                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
597                         return NULL;
598                 else
599                         swapping = FALSE;
600         }
601
602         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
603
604         if (folder_has_parent_of_type(item, F_QUEUE)) {
605                 tmp_flags |= MSG_QUEUED;
606         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
607                 tmp_flags |= MSG_DRAFT;
608         }
609
610         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0) {
611                 fclose(fp);
612                 return NULL;
613         }
614         dstcharset = CS_UTF_8;
615         if (srccharset == NULL || dstcharset == NULL) {
616                 conv = NULL;
617         } else if (strcmp(srccharset, dstcharset) == 0) {
618                 debug_print("using Noop Converter\n");
619
620                 conv = NULL;
621         } else {
622                 CharsetConverter *charsetconv;
623
624                 debug_print("using CharsetConverter\n");
625
626                 charsetconv = g_new0(CharsetConverter, 1);
627                 charsetconv->converter.convert = strconv_charset_convert;
628                 charsetconv->converter.free = strconv_charset_free;
629                 charsetconv->srccharset = g_strdup(srccharset);
630                 charsetconv->dstcharset = g_strdup(dstcharset);
631
632                 conv = (StringConverter *) charsetconv;
633         }
634         g_free(srccharset);
635
636         cache = msgcache_new();
637
638         if (msgcache_use_mmap_read == TRUE) {
639                 if (fstat(fileno(fp), &st) >= 0)
640                         map_len = st.st_size;
641                 else
642                         map_len = -1;
643                 if (map_len > 0) {
644 #ifdef G_OS_WIN32
645                         cache_data = NULL;
646                         HANDLE hFile, hMapping;
647                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
648                         if (hFile == (HANDLE) -1)
649                                 goto w32_fail;
650                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
651                         if (!hMapping)
652                                 goto w32_fail;
653                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
654                         CloseHandle (hMapping);
655                 w32_fail:
656                         ;
657 #else
658                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
659 #endif
660                 }
661         } else {
662                 cache_data = NULL;
663         }
664         if (cache_data != NULL && cache_data != MAP_FAILED) {
665                 int rem_len = map_len-ftell(fp);
666                 char *walk_data = cache_data+ftell(fp);
667
668                 while(rem_len > 0) {
669                         GET_CACHE_DATA_INT(num);
670                         
671                         msginfo = procmsg_msginfo_new();
672                         msginfo->msgnum = num;
673                         memusage += sizeof(MsgInfo);
674
675                         GET_CACHE_DATA_INT(msginfo->size);
676                         GET_CACHE_DATA_INT(msginfo->mtime);
677                         GET_CACHE_DATA_INT(msginfo->date_t);
678                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
679
680                         GET_CACHE_DATA(msginfo->fromname, memusage);
681
682                         GET_CACHE_DATA(msginfo->date, memusage);
683                         GET_CACHE_DATA(msginfo->from, memusage);
684                         GET_CACHE_DATA(msginfo->to, memusage);
685                         GET_CACHE_DATA(msginfo->cc, memusage);
686                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
687                         GET_CACHE_DATA(msginfo->subject, memusage);
688                         GET_CACHE_DATA(msginfo->msgid, memusage);
689                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
690                         GET_CACHE_DATA(msginfo->xref, memusage);
691
692                         GET_CACHE_DATA_INT(msginfo->planned_download);
693                         GET_CACHE_DATA_INT(msginfo->total_size);
694                         GET_CACHE_DATA_INT(refnum);
695
696                         for (; refnum != 0; refnum--) {
697                                 ref = NULL;
698
699                                 GET_CACHE_DATA(ref, memusage);
700
701                                 if (ref && *ref)
702                                         msginfo->references =
703                                                 g_slist_prepend(msginfo->references, ref);
704                         }
705                         if (msginfo->references)
706                                 msginfo->references =
707                                         g_slist_reverse(msginfo->references);
708
709                         msginfo->folder = item;
710                         msginfo->flags.tmp_flags |= tmp_flags;
711
712                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
713                         if(msginfo->msgid)
714                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
715                 }
716
717 #ifdef G_OS_WIN32
718                 UnmapViewOfFile((void*) cache_data);
719 #else
720                 munmap(cache_data, map_len);
721 #endif
722         } else {
723                 while (fread(&num, sizeof(num), 1, fp) == 1) {
724                         if (swapping)
725                                 num = bswap_32(num);
726
727                         msginfo = procmsg_msginfo_new();
728                         msginfo->msgnum = num;
729                         memusage += sizeof(MsgInfo);
730
731                         READ_CACHE_DATA_INT(msginfo->size, fp);
732                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
733                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
734                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
735
736                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
737
738                         READ_CACHE_DATA(msginfo->date, fp, memusage);
739                         READ_CACHE_DATA(msginfo->from, fp, memusage);
740                         READ_CACHE_DATA(msginfo->to, fp, memusage);
741                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
742                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
743                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
744                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
745                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
746                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
747
748                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
749                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
750                         READ_CACHE_DATA_INT(refnum, fp);
751
752                         for (; refnum != 0; refnum--) {
753                                 ref = NULL;
754
755                                 READ_CACHE_DATA(ref, fp, memusage);
756
757                                 if (ref && *ref)
758                                         msginfo->references =
759                                                 g_slist_prepend(msginfo->references, ref);
760                         }
761                         if (msginfo->references)
762                                 msginfo->references =
763                                         g_slist_reverse(msginfo->references);
764
765                         msginfo->folder = item;
766                         msginfo->flags.tmp_flags |= tmp_flags;
767
768                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
769                         if(msginfo->msgid)
770                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
771                 }
772         }
773 bail_err:
774         fclose(fp);
775
776         if (conv != NULL) {
777                 if (conv->free != NULL)
778                         conv->free(conv);
779                 g_free(conv);
780         }
781
782         if(error) {
783                 msgcache_destroy(cache);
784                 return NULL;
785         }
786
787         cache->last_access = time(NULL);
788         cache->memusage = memusage;
789
790         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
791         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
792
793         return cache;
794 }
795
796 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
797 {
798         FILE *fp;
799         MsgInfo *msginfo;
800         MsgPermFlags perm_flags;
801         guint32 num;
802         gint map_len = -1;
803         char *cache_data = NULL;
804         struct stat st;
805         gboolean error;
806
807         swapping = TRUE;
808
809         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
810          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
811          * it means it's the old version (not little-endian) on a big-endian machine. The code has
812          * no effect on x86 as their file doesn't change. */
813
814         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
815                 /* see if it isn't swapped ? */
816                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
817                         return;
818                 else
819                         swapping = FALSE; /* yay */
820         }
821         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
822         
823         if (msgcache_use_mmap_read) {
824                 if (fstat(fileno(fp), &st) >= 0)
825                         map_len = st.st_size;
826                 else
827                         map_len = -1;
828                 if (map_len > 0) {
829 #ifdef G_OS_WIN32
830                         cache_data = NULL;
831                         HANDLE hFile, hMapping;
832                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
833                         if (hFile == (HANDLE) -1)
834                                 goto w32_fail2;
835                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
836                         if (!hMapping)
837                                 goto w32_fail2;
838                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
839                         CloseHandle (hMapping);
840                 w32_fail2:
841                         ;
842 #else
843                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
844 #endif
845                 }
846         } else {
847                 cache_data = NULL;
848         }
849         if (cache_data != NULL && cache_data != MAP_FAILED) {
850                 int rem_len = map_len-ftell(fp);
851                 char *walk_data = cache_data+ftell(fp);
852
853                 while(rem_len > 0) {
854                         GET_CACHE_DATA_INT(num);
855                         GET_CACHE_DATA_INT(perm_flags);
856                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
857                         if(msginfo) {
858                                 msginfo->flags.perm_flags = perm_flags;
859                         }
860                 }
861 #ifdef G_OS_WIN32
862                 UnmapViewOfFile((void*) cache_data);
863 #else
864                 munmap(cache_data, map_len);
865 #endif
866         } else {
867                 while (fread(&num, sizeof(num), 1, fp) == 1) {
868                         if (swapping)
869                                 num = bswap_32(num);
870                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) break;
871                         if (swapping)
872                                 perm_flags = bswap_32(perm_flags);
873                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
874                         if(msginfo) {
875                                 msginfo->flags.perm_flags = perm_flags;
876                         }
877                 }       
878         }
879 bail_err:
880         fclose(fp);
881 }
882
883 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
884 {
885         FILE *fp;
886         MsgInfo *msginfo;
887         guint32 num;
888         gint map_len = -1;
889         char *cache_data = NULL;
890         struct stat st;
891         gboolean error = FALSE;
892
893         swapping = TRUE;
894
895         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
896          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
897          * it means it's the old version (not little-endian) on a big-endian machine. The code has
898          * no effect on x86 as their file doesn't change. */
899
900         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
901                 /* see if it isn't swapped ? */
902                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
903                         return;
904                 else
905                         swapping = FALSE; /* yay */
906         }
907         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
908         
909         if (msgcache_use_mmap_read) {
910                 if (fstat(fileno(fp), &st) >= 0)
911                         map_len = st.st_size;
912                 else
913                         map_len = -1;
914                 if (map_len > 0) {
915 #ifdef G_OS_WIN32
916                         cache_data = NULL;
917                         HANDLE hFile, hMapping;
918                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
919                         if (hFile == (HANDLE) -1)
920                                 goto w32_fail6;
921                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
922                         if (!hMapping)
923                                 goto w32_fail6;
924                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
925                         CloseHandle (hMapping);
926                 w32_fail6:
927                         ;
928 #else
929                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
930 #endif
931                 }
932         } else {
933                 cache_data = NULL;
934         }
935         if (cache_data != NULL && cache_data != MAP_FAILED) {
936                 int rem_len = map_len-ftell(fp);
937                 char *walk_data = cache_data+ftell(fp);
938
939                 while(rem_len > 0) {
940                         gint id = -1;
941                         GET_CACHE_DATA_INT(num);
942                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
943                         if(msginfo) {
944                                 g_slist_free(msginfo->tags);
945                                 msginfo->tags = NULL;
946                                 do {
947                                         GET_CACHE_DATA_INT(id);
948                                         if (id > 0) {
949                                                 msginfo->tags = g_slist_prepend(
950                                                         msginfo->tags, 
951                                                         GINT_TO_POINTER(id));
952                                         }
953                                 } while (id > 0);
954                                 msginfo->tags = g_slist_reverse(msginfo->tags);
955                         }
956                 }
957 #ifdef G_OS_WIN32
958                 UnmapViewOfFile((void*) cache_data);
959 #else
960                 munmap(cache_data, map_len);
961 #endif
962         } else {
963                 while (fread(&num, sizeof(num), 1, fp) == 1) {
964                         gint id = -1;
965                         if (swapping)
966                                 num = bswap_32(num);
967                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
968                         if(msginfo) {
969                                 g_slist_free(msginfo->tags);
970                                 msginfo->tags = NULL;
971                                 do {
972                                         if (fread(&id, sizeof(id), 1, fp) != 1) 
973                                                 id = -1;
974                                         if (swapping)
975                                                 id = bswap_32(id);
976                                         if (id > 0) {
977                                                 msginfo->tags = g_slist_prepend(
978                                                         msginfo->tags, 
979                                                         GINT_TO_POINTER(id));
980                                         }
981                                 } while (id > 0);
982                                 msginfo->tags = g_slist_reverse(msginfo->tags);
983                         }
984                 }
985         }
986 bail_err:
987         fclose(fp);
988 }
989
990 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
991 {
992         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
993         GSList *cur;
994         int w_err = 0, wrote = 0;
995
996         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
997         WRITE_CACHE_DATA_INT(msginfo->size, fp);
998         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
999         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
1000         WRITE_CACHE_DATA_INT(flags, fp);
1001
1002         WRITE_CACHE_DATA(msginfo->fromname, fp);
1003
1004         WRITE_CACHE_DATA(msginfo->date, fp);
1005         WRITE_CACHE_DATA(msginfo->from, fp);
1006         WRITE_CACHE_DATA(msginfo->to, fp);
1007         WRITE_CACHE_DATA(msginfo->cc, fp);
1008         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
1009         WRITE_CACHE_DATA(msginfo->subject, fp);
1010         WRITE_CACHE_DATA(msginfo->msgid, fp);
1011         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
1012         WRITE_CACHE_DATA(msginfo->xref, fp);
1013         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
1014         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
1015         
1016         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
1017
1018         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
1019                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
1020         }
1021         return w_err ? -1 : wrote;
1022 }
1023
1024 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
1025 {
1026         MsgPermFlags flags = msginfo->flags.perm_flags;
1027         int w_err = 0, wrote = 0;
1028         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1029         WRITE_CACHE_DATA_INT(flags, fp);
1030         return w_err ? -1 : wrote;
1031 }
1032
1033 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
1034 {
1035         GSList *cur = msginfo->tags;
1036         int w_err = 0, wrote = 0;
1037
1038         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1039         for (; cur; cur = cur->next) {
1040                 gint id = GPOINTER_TO_INT(cur->data);
1041                 if (tags_get_tag(id) != NULL) {
1042                         WRITE_CACHE_DATA_INT(id, fp);
1043                 }
1044         }
1045         WRITE_CACHE_DATA_INT(-1, fp);
1046
1047         return w_err ? -1 : wrote;
1048 }
1049
1050 struct write_fps
1051 {
1052         FILE *cache_fp;
1053         FILE *mark_fp;
1054         FILE *tags_fp;
1055         int error;
1056         guint cache_size;
1057         guint mark_size;
1058         guint tags_size;
1059 };
1060
1061 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1062 {
1063         MsgInfo *msginfo;
1064         struct write_fps *write_fps;
1065         int tmp;
1066
1067         msginfo = (MsgInfo *)value;
1068         write_fps = user_data;
1069
1070         if (write_fps->cache_fp) {
1071                 tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1072                 if (tmp < 0)
1073                         write_fps->error = 1;
1074                 else
1075                         write_fps->cache_size += tmp;
1076         }
1077         if (write_fps->mark_fp) {
1078         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1079                 if (tmp < 0)
1080                         write_fps->error = 1;
1081                 else
1082                         write_fps->mark_size += tmp;
1083                 }
1084         if (write_fps->tags_fp) {
1085                 tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1086                 if (tmp < 0)
1087                         write_fps->error = 1;
1088                 else
1089                         write_fps->tags_size += tmp;
1090         }
1091 }
1092
1093 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1094 {
1095         struct write_fps write_fps;
1096         gchar *new_cache, *new_mark, *new_tags;
1097         int w_err = 0, wrote = 0;
1098
1099         START_TIMING("");
1100         cm_return_val_if_fail(cache != NULL, -1);
1101
1102         new_cache = g_strconcat(cache_file, ".new", NULL);
1103         new_mark  = g_strconcat(mark_file, ".new", NULL);
1104         new_tags  = g_strconcat(tags_file, ".new", NULL);
1105
1106         write_fps.error = 0;
1107         write_fps.cache_size = 0;
1108         write_fps.mark_size = 0;
1109         write_fps.tags_size = 0;
1110
1111         /* open files and write headers */
1112
1113         if (cache_file) {
1114                 write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1115                         DATA_WRITE, NULL, 0);
1116                 if (write_fps.cache_fp == NULL) {
1117                         g_free(new_cache);
1118                         g_free(new_mark);
1119                         g_free(new_tags);
1120                         return -1;
1121                 }
1122                 WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1123         } else {
1124                 write_fps.cache_fp = NULL;
1125         }
1126
1127         if (w_err != 0) {
1128                 g_warning("failed to write charset\n");
1129                 fclose(write_fps.cache_fp);
1130                 claws_unlink(new_cache);
1131                 g_free(new_cache);
1132                 g_free(new_mark);
1133                 g_free(new_tags);
1134                 return -1;
1135         }
1136
1137         if (mark_file) {
1138                 write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1139                         DATA_WRITE, NULL, 0);
1140                 if (write_fps.mark_fp == NULL) {
1141                         fclose(write_fps.cache_fp);
1142                         claws_unlink(new_cache);
1143                         g_free(new_cache);
1144                         g_free(new_mark);
1145                         g_free(new_tags);
1146                         return -1;
1147                 }
1148         } else {
1149                 write_fps.mark_fp = NULL;
1150         }
1151
1152         if (tags_file) {
1153                 write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1154                         DATA_WRITE, NULL, 0);
1155                 if (write_fps.tags_fp == NULL) {
1156                         fclose(write_fps.cache_fp);
1157                         fclose(write_fps.mark_fp);
1158                         claws_unlink(new_cache);
1159                         claws_unlink(new_mark);
1160                         g_free(new_cache);
1161                         g_free(new_mark);
1162                         g_free(new_tags);
1163                         return -1;
1164                 }
1165         } else {
1166                 write_fps.tags_fp = NULL;
1167         }
1168
1169         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1170
1171         if (write_fps.cache_fp && change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1172                 FILE_OP_ERROR(new_cache, "chmod");
1173
1174         /* headers written, note file size */
1175         if (write_fps.cache_fp)
1176                 write_fps.cache_size = ftell(write_fps.cache_fp);
1177         if (write_fps.mark_fp)
1178                 write_fps.mark_size = ftell(write_fps.mark_fp);
1179         if (write_fps.tags_fp)
1180                 write_fps.tags_size = ftell(write_fps.tags_fp);
1181
1182 #ifdef HAVE_FWRITE_UNLOCKED
1183         /* lock files for write once (instead of once per fwrite) */
1184         if (write_fps.cache_fp)
1185                 flockfile(write_fps.cache_fp);
1186         if (write_fps.mark_fp)
1187                 flockfile(write_fps.mark_fp);
1188         if (write_fps.tags_fp)
1189                 flockfile(write_fps.tags_fp);
1190 #endif
1191         /* write data to the files */
1192         g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1193 #ifdef HAVE_FWRITE_UNLOCKED
1194         /* unlock files */
1195         if (write_fps.cache_fp)
1196                 funlockfile(write_fps.cache_fp);
1197         if (write_fps.mark_fp)
1198                 funlockfile(write_fps.mark_fp);
1199         if (write_fps.tags_fp)
1200                 funlockfile(write_fps.tags_fp);
1201 #endif
1202         /* flush buffers */
1203         if (write_fps.cache_fp)
1204                 write_fps.error |= (fflush(write_fps.cache_fp) != 0);
1205         if (write_fps.mark_fp)
1206                 write_fps.error |= (fflush(write_fps.mark_fp) != 0);
1207         if (write_fps.tags_fp)
1208                 write_fps.error |= (fflush(write_fps.tags_fp) != 0);
1209
1210         /* sync to filesystem */
1211         if (prefs_common.flush_metadata && write_fps.cache_fp)
1212                 write_fps.error |= (fsync(fileno(write_fps.cache_fp)) != 0);
1213         if (prefs_common.flush_metadata && write_fps.mark_fp)
1214                 write_fps.error |= (fsync(fileno(write_fps.mark_fp)) != 0);
1215         if (prefs_common.flush_metadata && write_fps.tags_fp)
1216                 write_fps.error |= (fsync(fileno(write_fps.tags_fp)) != 0);
1217
1218         /* close files */
1219         if (write_fps.cache_fp)
1220                 write_fps.error |= (fclose(write_fps.cache_fp) != 0);
1221         if (write_fps.mark_fp)
1222                 write_fps.error |= (fclose(write_fps.mark_fp) != 0);
1223         if (write_fps.tags_fp)
1224                 write_fps.error |= (fclose(write_fps.tags_fp) != 0);
1225
1226
1227         if (write_fps.error != 0) {
1228                 /* in case of error, forget all */
1229                 claws_unlink(new_cache);
1230                 claws_unlink(new_mark);
1231                 claws_unlink(new_tags);
1232                 g_free(new_cache);
1233                 g_free(new_mark);
1234                 g_free(new_tags);
1235                 return -1;
1236         } else {
1237                 /* switch files */
1238                 if (cache_file)
1239                         move_file(new_cache, cache_file, TRUE);
1240                 if (mark_file)
1241                         move_file(new_mark, mark_file, TRUE);
1242                 if (tags_file)
1243                         move_file(new_tags, tags_file, TRUE);
1244                 cache->last_access = time(NULL);
1245         }
1246
1247         g_free(new_cache);
1248         g_free(new_mark);
1249         g_free(new_tags);
1250         debug_print("done.\n");
1251         END_TIMING();
1252         return 0;
1253 }
1254