2009-04-02 [colin] 3.7.1cvs33
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2009 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  * 
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #endif
23
24 #include "defs.h"
25
26 #define _GNU_SOURCE
27 #include <stdio.h>
28
29 #include <glib.h>
30 #include <glib/gi18n.h>
31 #ifdef _WIN32
32 # include <w32lib.h>
33 # define MAP_FAILED     ((char *) -1)
34 #else
35 # include <sys/mman.h>
36 #endif
37 #include <sys/types.h>
38 #include <sys/stat.h>
39
40 #include <time.h>
41
42 #include "msgcache.h"
43 #include "utils.h"
44 #include "procmsg.h"
45 #include "codeconv.h"
46 #include "timing.h"
47 #include "tags.h"
48 #include "prefs_common.h"
49
50 #ifdef HAVE_FWRITE_UNLOCKED
51 #define SC_FWRITE fwrite_unlocked
52 #else
53 #define SC_FWRITE fwrite
54 #endif
55
56 #if G_BYTE_ORDER == G_BIG_ENDIAN
57 #define bswap_32(x) \
58      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
59       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
60      
61 #define MMAP_TO_GUINT32(x)      \
62         (((x[3]&0xff)) |        \
63          ((x[2]&0xff) << 8) |   \
64          ((x[1]&0xff) << 16) |  \
65          ((x[0]&0xff) << 24))
66
67 #define MMAP_TO_GUINT32_SWAPPED(x)      \
68         (((x[0]&0xff)) |                \
69          ((x[1]&0xff) << 8) |           \
70          ((x[2]&0xff) << 16) |          \
71          ((x[3]&0xff) << 24))
72
73 static gboolean msgcache_use_mmap_read = TRUE;
74
75 #else
76 #define bswap_32(x) (x)
77
78 #define MMAP_TO_GUINT32(x)      \
79         (((x[0]&0xff)) |        \
80          ((x[1]&0xff) << 8) |   \
81          ((x[2]&0xff) << 16) |  \
82          ((x[3]&0xff) << 24))
83
84 #define MMAP_TO_GUINT32_SWAPPED(x)      \
85         (((x[0]&0xff)) |                \
86          ((x[1]&0xff) << 8) |           \
87          ((x[2]&0xff) << 16) |          \
88          ((x[3]&0xff) << 24))
89
90 static gboolean msgcache_use_mmap_read = TRUE;
91 #endif
92
93 static gboolean swapping = TRUE;
94
95 typedef enum
96 {
97         DATA_READ,
98         DATA_WRITE,
99         DATA_APPEND
100 } DataOpenMode;
101
102 struct _MsgCache {
103         GHashTable      *msgnum_table;
104         GHashTable      *msgid_table;
105         guint            memusage;
106         time_t           last_access;
107 };
108
109 typedef struct _StringConverter StringConverter;
110 struct _StringConverter {
111         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
112         void   (*free)    (StringConverter *converter);
113 };
114
115 typedef struct _StrdupConverter StrdupConverter;
116 struct _StrdupConverter {
117         StringConverter converter;
118 };
119
120 typedef struct _CharsetConverter CharsetConverter;
121 struct _CharsetConverter {
122         StringConverter converter;
123
124         gchar *srccharset;
125         gchar *dstcharset;
126 };
127
128 MsgCache *msgcache_new(void)
129 {
130         MsgCache *cache;
131         
132         cache = g_new0(MsgCache, 1),
133         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
134         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
135         cache->last_access = time(NULL);
136
137         return cache;
138 }
139
140 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
141 {
142         procmsg_msginfo_free((MsgInfo *)msginfo);
143         return TRUE;
144 }                                                                                         
145
146 void msgcache_destroy(MsgCache *cache)
147 {
148         cm_return_if_fail(cache != NULL);
149
150         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
151         g_hash_table_destroy(cache->msgid_table);
152         g_hash_table_destroy(cache->msgnum_table);
153         g_free(cache);
154 }
155
156 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
157 {
158         MsgInfo *newmsginfo;
159
160         cm_return_if_fail(cache != NULL);
161         cm_return_if_fail(msginfo != NULL);
162
163         newmsginfo = procmsg_msginfo_new_ref(msginfo);
164         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
165         if(newmsginfo->msgid != NULL)
166                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
167         cache->memusage += procmsg_msginfo_memusage(msginfo);
168         cache->last_access = time(NULL);
169
170         msginfo->folder->cache_dirty = TRUE;
171
172         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
173 }
174
175 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
176 {
177         MsgInfo *msginfo;
178
179         cm_return_if_fail(cache != NULL);
180         cm_return_if_fail(msgnum > 0);
181
182         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
183         if(!msginfo)
184                 return;
185
186         cache->memusage -= procmsg_msginfo_memusage(msginfo);
187         if(msginfo->msgid)
188                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
189         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
190         procmsg_msginfo_free(msginfo);
191         cache->last_access = time(NULL);
192
193         msginfo->folder->cache_dirty = TRUE;
194
195         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
196 }
197
198 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
199 {
200         MsgInfo *oldmsginfo, *newmsginfo;
201         
202         cm_return_if_fail(cache != NULL);
203         cm_return_if_fail(msginfo != NULL);
204
205         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
206         if(oldmsginfo && oldmsginfo->msgid) 
207                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
208         if (oldmsginfo) {
209                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
210                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
211                 procmsg_msginfo_free(oldmsginfo);
212         }
213
214         newmsginfo = procmsg_msginfo_new_ref(msginfo);
215         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
216         if(newmsginfo->msgid)
217                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
218         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
219         cache->last_access = time(NULL);
220         
221         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
222
223         msginfo->folder->cache_dirty = TRUE;
224
225         return;
226 }
227
228 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
229 {
230         MsgInfo *msginfo;
231
232         cm_return_val_if_fail(cache != NULL, NULL);
233
234         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
235         if(!msginfo)
236                 return NULL;
237         cache->last_access = time(NULL);
238         
239         return procmsg_msginfo_new_ref(msginfo);
240 }
241
242 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
243 {
244         MsgInfo *msginfo;
245         
246         cm_return_val_if_fail(cache != NULL, NULL);
247         cm_return_val_if_fail(msgid != NULL, NULL);
248
249         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
250         if(!msginfo)
251                 return NULL;
252         cache->last_access = time(NULL);
253         
254         return procmsg_msginfo_new_ref(msginfo);        
255 }
256
257 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
258 {
259         MsgInfoList **listptr = user_data;
260         MsgInfo *msginfo = value;
261
262         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
263 }
264
265 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
266 {
267         MsgInfoList *msg_list = NULL;
268         START_TIMING("");
269         cm_return_val_if_fail(cache != NULL, NULL);
270
271         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
272         cache->last_access = time(NULL);
273         
274         msg_list = g_slist_reverse(msg_list);
275         END_TIMING();
276         return msg_list;
277 }
278
279 time_t msgcache_get_last_access_time(MsgCache *cache)
280 {
281         cm_return_val_if_fail(cache != NULL, 0);
282         
283         return cache->last_access;
284 }
285
286 gint msgcache_get_memory_usage(MsgCache *cache)
287 {
288         cm_return_val_if_fail(cache != NULL, 0);
289
290         return cache->memusage;
291 }
292
293 /*
294  *  Cache saving functions
295  */
296
297 #define READ_CACHE_DATA(data, fp, total_len) \
298 { \
299         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
300                 procmsg_msginfo_free(msginfo); \
301                 error = TRUE; \
302                 goto bail_err; \
303         } \
304         total_len += tmp_len; \
305 }
306
307 #define READ_CACHE_DATA_INT(n, fp) \
308 { \
309         guint32 idata; \
310         size_t ni; \
311  \
312         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
313                 g_warning("read_int: Cache data corrupted, read %zd of %zd at " \
314                           "offset %ld\n", ni, sizeof(idata), ftell(fp)); \
315                 procmsg_msginfo_free(msginfo); \
316                 error = TRUE; \
317                 goto bail_err; \
318         } else \
319                 n = swapping ? bswap_32(idata) : (idata);\
320 }
321
322 #define GET_CACHE_DATA_INT(n) \
323 { \
324         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data))); \
325         walk_data += 4; rem_len -= 4;                   \
326 }
327
328 #define GET_CACHE_DATA(data, total_len) \
329 { \
330         GET_CACHE_DATA_INT(tmp_len);    \
331         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
332                 g_print("error at rem_len:%d\n", rem_len);\
333                 procmsg_msginfo_free(msginfo); \
334                 error = TRUE; \
335                 goto bail_err; \
336         } \
337         total_len += tmp_len; \
338         walk_data += tmp_len; rem_len -= tmp_len; \
339 }
340
341
342 #define WRITE_CACHE_DATA_INT(n, fp)                     \
343 {                                                       \
344         guint32 idata;                                  \
345                                                         \
346         idata = (guint32)bswap_32(n);                   \
347         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
348                 w_err = 1;                              \
349         wrote += 4;                                     \
350 }
351
352 #define PUT_CACHE_DATA_INT(n)                           \
353 {                                                       \
354         walk_data[0]=(((guint32)n)&0x000000ff);                 \
355         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
356         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
357         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
358         walk_data += 4;                                 \
359         wrote += 4;                                     \
360 }
361
362 #define WRITE_CACHE_DATA(data, fp) \
363 { \
364         size_t len;                                     \
365         if (data == NULL)                               \
366                 len = 0;                                \
367         else                                            \
368                 len = strlen(data);                     \
369         WRITE_CACHE_DATA_INT(len, fp);                  \
370         if (w_err == 0 && len > 0) {                    \
371                 if (SC_FWRITE(data, 1, len, fp) != len) \
372                         w_err = 1;                      \
373                 wrote += len;                           \
374         } \
375 }
376
377 #define PUT_CACHE_DATA(data)                            \
378 {                                                       \
379         size_t len;                                     \
380         if (data == NULL)                               \
381                 len = 0;                                \
382         else                                            \
383                 len = strlen(data);                     \
384         PUT_CACHE_DATA_INT(len);                        \
385         if (len > 0) {                                  \
386                 memcpy(walk_data, data, len);           \
387                 walk_data += len;                       \
388                 wrote += len;                           \
389         }                                               \
390 }
391
392 static FILE *msgcache_open_data_file(const gchar *file, guint version,
393                                      DataOpenMode mode,
394                                      gchar *buf, size_t buf_size)
395 {
396         FILE *fp;
397         gint32 data_ver;
398
399         cm_return_val_if_fail(file != NULL, NULL);
400
401         if (mode == DATA_WRITE) {
402                 int w_err = 0, wrote = 0;
403                 if ((fp = g_fopen(file, "wb")) == NULL) {
404                         FILE_OP_ERROR(file, "fopen");
405                         return NULL;
406                 }
407                 if (change_file_mode_rw(fp, file) < 0)
408                         FILE_OP_ERROR(file, "chmod");
409
410                 WRITE_CACHE_DATA_INT(version, fp);
411                 if (w_err != 0) {
412                         g_warning("failed to write int\n");
413                         fclose(fp);
414                         return NULL;
415                 }
416                 return fp;
417         }
418
419         /* check version */
420         if ((fp = g_fopen(file, "rb")) == NULL)
421                 debug_print("Mark/Cache file '%s' not found\n", file);
422         else {
423                 if (buf && buf_size > 0)
424                         setvbuf(fp, buf, _IOFBF, buf_size);
425                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
426                          version != bswap_32(data_ver)) {
427                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
428                                   file, bswap_32(data_ver), version);
429                         fclose(fp);
430                         fp = NULL;
431                 }
432                 data_ver = bswap_32(data_ver);
433         }
434         
435         if (mode == DATA_READ)
436                 return fp;
437
438         if (fp) {
439                 /* reopen with append mode */
440                 fclose(fp);
441                 if ((fp = g_fopen(file, "ab")) == NULL)
442                         FILE_OP_ERROR(file, "fopen");
443         } else {
444                 /* open with overwrite mode if mark file doesn't exist or
445                    version is different */
446                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
447                                             buf_size);
448         }
449
450         return fp;
451 }
452
453 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
454                                          StringConverter *conv)
455 {
456         gchar *tmpstr = NULL;
457         size_t ni;
458         guint32 len;
459
460         *str = NULL;
461         if (!swapping) {
462                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
463                     len > G_MAXINT) {
464                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
465                                   "of %zd bytes at offset %ld\n", ni, sizeof(len), 
466                                   ftell(fp));
467                         return -1;
468                 }
469         } else {
470                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
471                     bswap_32(len) > G_MAXINT) {
472                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
473                                   "of %zd bytes at offset %ld\n", ni, sizeof(len), 
474                                   ftell(fp));
475                         return -1;
476                 }
477                 len = bswap_32(len);
478         }
479
480         if (len == 0)
481                 return 0;
482
483         tmpstr = g_try_malloc(len + 1);
484
485         if(tmpstr == NULL) {
486                 return -1;
487         }
488
489         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
490                 g_warning("read_data_str: Cache data corrupted, read %zd of %u "
491                           "bytes at offset %ld\n", 
492                           ni, len, ftell(fp));
493                 g_free(tmpstr);
494                 return -1;
495         }
496         tmpstr[len] = 0;
497
498         if (conv != NULL) {
499                 *str = conv->convert(conv, tmpstr);
500                 g_free(tmpstr);
501         } else 
502                 *str = tmpstr;
503
504         return len;
505 }
506
507 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
508                                          StringConverter *conv)
509 {
510         gchar *tmpstr = NULL;
511
512         *str = NULL;
513
514         if (len == 0)
515                 return 0;
516
517         if(len > 2*1024*1024) {
518                 g_warning("read_data_str: refusing to allocate %d bytes.\n", len);
519                 return -1;
520         }
521
522         tmpstr = g_try_malloc(len + 1);
523
524         if(tmpstr == NULL) {
525                 return -1;
526         }
527
528         memcpy(tmpstr, src, len);
529         tmpstr[len] = 0;
530
531         if (conv != NULL) {
532                 *str = conv->convert(conv, tmpstr);
533                 g_free(tmpstr);
534         } else 
535                 *str = tmpstr;
536
537         return len;
538 }
539
540 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
541 {
542         CharsetConverter *charsetconv = (CharsetConverter *) conv;
543
544         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
545 }
546
547 static void strconv_charset_free(StringConverter *conv)
548 {
549         CharsetConverter *charsetconv = (CharsetConverter *) conv;
550
551         g_free(charsetconv->srccharset);
552         g_free(charsetconv->dstcharset);
553 }
554
555 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
556 {
557         MsgCache *cache;
558         FILE *fp;
559         MsgInfo *msginfo;
560         MsgTmpFlags tmp_flags = 0;
561         gchar file_buf[BUFFSIZE];
562         guint32 num;
563         guint refnum;
564         gboolean error = FALSE;
565         StringConverter *conv = NULL;
566         gchar *srccharset = NULL;
567         const gchar *dstcharset = NULL;
568         gchar *ref = NULL;
569         guint memusage = 0;
570         gint tmp_len = 0, map_len = -1;
571         char *cache_data = NULL;
572         struct stat st;
573
574         cm_return_val_if_fail(cache_file != NULL, NULL);
575         cm_return_val_if_fail(item != NULL, NULL);
576
577         swapping = TRUE;
578
579         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
580          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
581          * it means it's the old version (not little-endian) on a big-endian machine. The code has
582          * no effect on x86 as their file doesn't change. */
583
584         if ((fp = msgcache_open_data_file
585                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
586                 if ((fp = msgcache_open_data_file
587                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
588                         return NULL;
589                 else
590                         swapping = FALSE;
591         }
592
593         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
594
595         if (folder_has_parent_of_type(item, F_QUEUE)) {
596                 tmp_flags |= MSG_QUEUED;
597         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
598                 tmp_flags |= MSG_DRAFT;
599         }
600
601         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0) {
602                 fclose(fp);
603                 return NULL;
604         }
605         dstcharset = CS_UTF_8;
606         if (srccharset == NULL || dstcharset == NULL) {
607                 conv = NULL;
608         } else if (strcmp(srccharset, dstcharset) == 0) {
609                 debug_print("using Noop Converter\n");
610
611                 conv = NULL;
612         } else {
613                 CharsetConverter *charsetconv;
614
615                 debug_print("using CharsetConverter\n");
616
617                 charsetconv = g_new0(CharsetConverter, 1);
618                 charsetconv->converter.convert = strconv_charset_convert;
619                 charsetconv->converter.free = strconv_charset_free;
620                 charsetconv->srccharset = g_strdup(srccharset);
621                 charsetconv->dstcharset = g_strdup(dstcharset);
622
623                 conv = (StringConverter *) charsetconv;
624         }
625         g_free(srccharset);
626
627         cache = msgcache_new();
628
629         if (msgcache_use_mmap_read == TRUE) {
630                 if (fstat(fileno(fp), &st) >= 0)
631                         map_len = st.st_size;
632                 else
633                         map_len = -1;
634                 if (map_len > 0) {
635 #ifdef G_OS_WIN32
636                         cache_data = NULL;
637                         HANDLE hFile, hMapping;
638                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
639                         if (hFile == (HANDLE) -1)
640                                 goto w32_fail;
641                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
642                         if (!hMapping)
643                                 goto w32_fail;
644                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
645                         CloseHandle (hMapping);
646                 w32_fail:
647                         ;
648 #else
649                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
650 #endif
651                 }
652         } else {
653                 cache_data = NULL;
654         }
655         if (cache_data != NULL && cache_data != MAP_FAILED) {
656                 int rem_len = map_len-ftell(fp);
657                 char *walk_data = cache_data+ftell(fp);
658
659                 while(rem_len > 0) {
660                         GET_CACHE_DATA_INT(num);
661                         
662                         msginfo = procmsg_msginfo_new();
663                         msginfo->msgnum = num;
664                         memusage += sizeof(MsgInfo);
665
666                         GET_CACHE_DATA_INT(msginfo->size);
667                         GET_CACHE_DATA_INT(msginfo->mtime);
668                         GET_CACHE_DATA_INT(msginfo->date_t);
669                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
670
671                         GET_CACHE_DATA(msginfo->fromname, memusage);
672
673                         GET_CACHE_DATA(msginfo->date, memusage);
674                         GET_CACHE_DATA(msginfo->from, memusage);
675                         GET_CACHE_DATA(msginfo->to, memusage);
676                         GET_CACHE_DATA(msginfo->cc, memusage);
677                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
678                         GET_CACHE_DATA(msginfo->subject, memusage);
679                         GET_CACHE_DATA(msginfo->msgid, memusage);
680                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
681                         GET_CACHE_DATA(msginfo->xref, memusage);
682
683                         GET_CACHE_DATA_INT(msginfo->planned_download);
684                         GET_CACHE_DATA_INT(msginfo->total_size);
685                         GET_CACHE_DATA_INT(refnum);
686
687                         for (; refnum != 0; refnum--) {
688                                 ref = NULL;
689
690                                 GET_CACHE_DATA(ref, memusage);
691
692                                 if (ref && *ref)
693                                         msginfo->references =
694                                                 g_slist_prepend(msginfo->references, ref);
695                         }
696                         if (msginfo->references)
697                                 msginfo->references =
698                                         g_slist_reverse(msginfo->references);
699
700                         msginfo->folder = item;
701                         msginfo->flags.tmp_flags |= tmp_flags;
702
703                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
704                         if(msginfo->msgid)
705                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
706                 }
707
708 #ifdef G_OS_WIN32
709                 UnmapViewOfFile((void*) cache_data);
710 #else
711                 munmap(cache_data, map_len);
712 #endif
713         } else {
714                 while (fread(&num, sizeof(num), 1, fp) == 1) {
715                         if (swapping)
716                                 num = bswap_32(num);
717
718                         msginfo = procmsg_msginfo_new();
719                         msginfo->msgnum = num;
720                         memusage += sizeof(MsgInfo);
721
722                         READ_CACHE_DATA_INT(msginfo->size, fp);
723                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
724                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
725                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
726
727                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
728
729                         READ_CACHE_DATA(msginfo->date, fp, memusage);
730                         READ_CACHE_DATA(msginfo->from, fp, memusage);
731                         READ_CACHE_DATA(msginfo->to, fp, memusage);
732                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
733                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
734                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
735                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
736                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
737                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
738
739                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
740                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
741                         READ_CACHE_DATA_INT(refnum, fp);
742
743                         for (; refnum != 0; refnum--) {
744                                 ref = NULL;
745
746                                 READ_CACHE_DATA(ref, fp, memusage);
747
748                                 if (ref && *ref)
749                                         msginfo->references =
750                                                 g_slist_prepend(msginfo->references, ref);
751                         }
752                         if (msginfo->references)
753                                 msginfo->references =
754                                         g_slist_reverse(msginfo->references);
755
756                         msginfo->folder = item;
757                         msginfo->flags.tmp_flags |= tmp_flags;
758
759                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
760                         if(msginfo->msgid)
761                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
762                 }
763         }
764 bail_err:
765         fclose(fp);
766
767         if (conv != NULL) {
768                 if (conv->free != NULL)
769                         conv->free(conv);
770                 g_free(conv);
771         }
772
773         if(error) {
774                 msgcache_destroy(cache);
775                 return NULL;
776         }
777
778         cache->last_access = time(NULL);
779         cache->memusage = memusage;
780
781         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
782         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
783
784         return cache;
785 }
786
787 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
788 {
789         FILE *fp;
790         MsgInfo *msginfo;
791         MsgPermFlags perm_flags;
792         guint32 num;
793         gint map_len = -1;
794         char *cache_data = NULL;
795         struct stat st;
796         
797         swapping = TRUE;
798
799         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
800          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
801          * it means it's the old version (not little-endian) on a big-endian machine. The code has
802          * no effect on x86 as their file doesn't change. */
803
804         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
805                 /* see if it isn't swapped ? */
806                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
807                         return;
808                 else
809                         swapping = FALSE; /* yay */
810         }
811         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
812         
813         if (msgcache_use_mmap_read) {
814                 if (fstat(fileno(fp), &st) >= 0)
815                         map_len = st.st_size;
816                 else
817                         map_len = -1;
818                 if (map_len > 0) {
819 #ifdef G_OS_WIN32
820                         cache_data = NULL;
821                         HANDLE hFile, hMapping;
822                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
823                         if (hFile == (HANDLE) -1)
824                                 goto w32_fail2;
825                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
826                         if (!hMapping)
827                                 goto w32_fail2;
828                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
829                         CloseHandle (hMapping);
830                 w32_fail2:
831                         ;
832 #else
833                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
834 #endif
835                 }
836         } else {
837                 cache_data = NULL;
838         }
839         if (cache_data != NULL && cache_data != MAP_FAILED) {
840                 int rem_len = map_len-ftell(fp);
841                 char *walk_data = cache_data+ftell(fp);
842
843                 while(rem_len > 0) {
844                         GET_CACHE_DATA_INT(num);
845                         GET_CACHE_DATA_INT(perm_flags);
846                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
847                         if(msginfo) {
848                                 msginfo->flags.perm_flags = perm_flags;
849                         }
850                 }
851 #ifdef G_OS_WIN32
852                 UnmapViewOfFile((void*) cache_data);
853 #else
854                 munmap(cache_data, map_len);
855 #endif
856         } else {
857                 while (fread(&num, sizeof(num), 1, fp) == 1) {
858                         if (swapping)
859                                 num = bswap_32(num);
860                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) break;
861                         if (swapping)
862                                 perm_flags = bswap_32(perm_flags);
863                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
864                         if(msginfo) {
865                                 msginfo->flags.perm_flags = perm_flags;
866                         }
867                 }       
868         }
869         fclose(fp);
870 }
871
872 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
873 {
874         FILE *fp;
875         MsgInfo *msginfo;
876         guint32 num;
877         gint map_len = -1;
878         char *cache_data = NULL;
879         struct stat st;
880         
881         swapping = TRUE;
882
883         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
884          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
885          * it means it's the old version (not little-endian) on a big-endian machine. The code has
886          * no effect on x86 as their file doesn't change. */
887
888         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
889                 /* see if it isn't swapped ? */
890                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
891                         return;
892                 else
893                         swapping = FALSE; /* yay */
894         }
895         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
896         
897         if (msgcache_use_mmap_read) {
898                 if (fstat(fileno(fp), &st) >= 0)
899                         map_len = st.st_size;
900                 else
901                         map_len = -1;
902                 if (map_len > 0) {
903 #ifdef G_OS_WIN32
904                         cache_data = NULL;
905                         HANDLE hFile, hMapping;
906                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
907                         if (hFile == (HANDLE) -1)
908                                 goto w32_fail6;
909                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
910                         if (!hMapping)
911                                 goto w32_fail6;
912                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
913                         CloseHandle (hMapping);
914                 w32_fail6:
915                         ;
916 #else
917                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
918 #endif
919                 }
920         } else {
921                 cache_data = NULL;
922         }
923         if (cache_data != NULL && cache_data != MAP_FAILED) {
924                 int rem_len = map_len-ftell(fp);
925                 char *walk_data = cache_data+ftell(fp);
926
927                 while(rem_len > 0) {
928                         gint id = -1;
929                         GET_CACHE_DATA_INT(num);
930                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
931                         if(msginfo) {
932                                 g_slist_free(msginfo->tags);
933                                 msginfo->tags = NULL;
934                                 do {
935                                         GET_CACHE_DATA_INT(id);
936                                         if (id > 0) {
937                                                 msginfo->tags = g_slist_prepend(
938                                                         msginfo->tags, 
939                                                         GINT_TO_POINTER(id));
940                                         }
941                                 } while (id > 0);
942                                 msginfo->tags = g_slist_reverse(msginfo->tags);
943                         }
944                 }
945 #ifdef G_OS_WIN32
946                 UnmapViewOfFile((void*) cache_data);
947 #else
948                 munmap(cache_data, map_len);
949 #endif
950         } else {
951                 while (fread(&num, sizeof(num), 1, fp) == 1) {
952                         gint id = -1;
953                         if (swapping)
954                                 num = bswap_32(num);
955                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
956                         if(msginfo) {
957                                 g_slist_free(msginfo->tags);
958                                 msginfo->tags = NULL;
959                                 do {
960                                         if (fread(&id, sizeof(id), 1, fp) != 1) 
961                                                 id = -1;
962                                         if (swapping)
963                                                 id = bswap_32(id);
964                                         if (id > 0) {
965                                                 msginfo->tags = g_slist_prepend(
966                                                         msginfo->tags, 
967                                                         GINT_TO_POINTER(id));
968                                         }
969                                 } while (id > 0);
970                                 msginfo->tags = g_slist_reverse(msginfo->tags);
971                         }
972                 }
973         }
974         fclose(fp);
975 }
976
977 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
978 {
979         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
980         GSList *cur;
981         int w_err = 0, wrote = 0;
982
983         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
984         WRITE_CACHE_DATA_INT(msginfo->size, fp);
985         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
986         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
987         WRITE_CACHE_DATA_INT(flags, fp);
988
989         WRITE_CACHE_DATA(msginfo->fromname, fp);
990
991         WRITE_CACHE_DATA(msginfo->date, fp);
992         WRITE_CACHE_DATA(msginfo->from, fp);
993         WRITE_CACHE_DATA(msginfo->to, fp);
994         WRITE_CACHE_DATA(msginfo->cc, fp);
995         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
996         WRITE_CACHE_DATA(msginfo->subject, fp);
997         WRITE_CACHE_DATA(msginfo->msgid, fp);
998         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
999         WRITE_CACHE_DATA(msginfo->xref, fp);
1000         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
1001         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
1002         
1003         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
1004
1005         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
1006                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
1007         }
1008         return w_err ? -1 : wrote;
1009 }
1010
1011 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
1012 {
1013         MsgPermFlags flags = msginfo->flags.perm_flags;
1014         int w_err = 0, wrote = 0;
1015         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1016         WRITE_CACHE_DATA_INT(flags, fp);
1017         return w_err ? -1 : wrote;
1018 }
1019
1020 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
1021 {
1022         GSList *cur = msginfo->tags;
1023         int w_err = 0, wrote = 0;
1024
1025         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1026         for (; cur; cur = cur->next) {
1027                 gint id = GPOINTER_TO_INT(cur->data);
1028                 if (tags_get_tag(id) != NULL) {
1029                         WRITE_CACHE_DATA_INT(id, fp);
1030                 }
1031         }
1032         WRITE_CACHE_DATA_INT(-1, fp);
1033
1034         return w_err ? -1 : wrote;
1035 }
1036
1037 struct write_fps
1038 {
1039         FILE *cache_fp;
1040         FILE *mark_fp;
1041         FILE *tags_fp;
1042         int error;
1043         guint cache_size;
1044         guint mark_size;
1045         guint tags_size;
1046 };
1047
1048 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1049 {
1050         MsgInfo *msginfo;
1051         struct write_fps *write_fps;
1052         int tmp;
1053
1054         msginfo = (MsgInfo *)value;
1055         write_fps = user_data;
1056
1057         if (write_fps->cache_fp) {
1058                 tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1059                 if (tmp < 0)
1060                         write_fps->error = 1;
1061                 else
1062                         write_fps->cache_size += tmp;
1063         }
1064         if (write_fps->mark_fp) {
1065         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1066                 if (tmp < 0)
1067                         write_fps->error = 1;
1068                 else
1069                         write_fps->mark_size += tmp;
1070                 }
1071         if (write_fps->tags_fp) {
1072                 tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1073                 if (tmp < 0)
1074                         write_fps->error = 1;
1075                 else
1076                         write_fps->tags_size += tmp;
1077         }
1078 }
1079
1080 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1081 {
1082         struct write_fps write_fps;
1083         gchar *new_cache, *new_mark, *new_tags;
1084         int w_err = 0, wrote = 0;
1085
1086         START_TIMING("");
1087         cm_return_val_if_fail(cache != NULL, -1);
1088
1089         new_cache = g_strconcat(cache_file, ".new", NULL);
1090         new_mark  = g_strconcat(mark_file, ".new", NULL);
1091         new_tags  = g_strconcat(tags_file, ".new", NULL);
1092
1093         write_fps.error = 0;
1094         write_fps.cache_size = 0;
1095         write_fps.mark_size = 0;
1096         write_fps.tags_size = 0;
1097
1098         /* open files and write headers */
1099
1100         if (cache_file) {
1101                 write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1102                         DATA_WRITE, NULL, 0);
1103                 if (write_fps.cache_fp == NULL) {
1104                         g_free(new_cache);
1105                         g_free(new_mark);
1106                         g_free(new_tags);
1107                         return -1;
1108                 }
1109                 WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1110         } else {
1111                 write_fps.cache_fp = NULL;
1112         }
1113
1114         if (w_err != 0) {
1115                 g_warning("failed to write charset\n");
1116                 fclose(write_fps.cache_fp);
1117                 claws_unlink(new_cache);
1118                 g_free(new_cache);
1119                 g_free(new_mark);
1120                 g_free(new_tags);
1121                 return -1;
1122         }
1123
1124         if (mark_file) {
1125                 write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1126                         DATA_WRITE, NULL, 0);
1127                 if (write_fps.mark_fp == NULL) {
1128                         fclose(write_fps.cache_fp);
1129                         claws_unlink(new_cache);
1130                         g_free(new_cache);
1131                         g_free(new_mark);
1132                         g_free(new_tags);
1133                         return -1;
1134                 }
1135         } else {
1136                 write_fps.mark_fp = NULL;
1137         }
1138
1139         if (tags_file) {
1140                 write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1141                         DATA_WRITE, NULL, 0);
1142                 if (write_fps.tags_fp == NULL) {
1143                         fclose(write_fps.cache_fp);
1144                         fclose(write_fps.mark_fp);
1145                         claws_unlink(new_cache);
1146                         claws_unlink(new_mark);
1147                         g_free(new_cache);
1148                         g_free(new_mark);
1149                         g_free(new_tags);
1150                         return -1;
1151                 }
1152         } else {
1153                 write_fps.tags_fp = NULL;
1154         }
1155
1156         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1157
1158         if (write_fps.cache_fp && change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1159                 FILE_OP_ERROR(new_cache, "chmod");
1160
1161         /* headers written, note file size */
1162         if (write_fps.cache_fp)
1163                 write_fps.cache_size = ftell(write_fps.cache_fp);
1164         if (write_fps.mark_fp)
1165                 write_fps.mark_size = ftell(write_fps.mark_fp);
1166         if (write_fps.tags_fp)
1167                 write_fps.tags_size = ftell(write_fps.tags_fp);
1168
1169 #ifdef HAVE_FWRITE_UNLOCKED
1170         /* lock files for write once (instead of once per fwrite) */
1171         if (write_fps.cache_fp)
1172                 flockfile(write_fps.cache_fp);
1173         if (write_fps.mark_fp)
1174                 flockfile(write_fps.mark_fp);
1175         if (write_fps.tags_fp)
1176                 flockfile(write_fps.tags_fp);
1177 #endif
1178         /* write data to the files */
1179         g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1180 #ifdef HAVE_FWRITE_UNLOCKED
1181         /* unlock files */
1182         if (write_fps.cache_fp)
1183                 funlockfile(write_fps.cache_fp);
1184         if (write_fps.mark_fp)
1185                 funlockfile(write_fps.mark_fp);
1186         if (write_fps.tags_fp)
1187                 funlockfile(write_fps.tags_fp);
1188 #endif
1189         /* flush buffers */
1190         if (write_fps.cache_fp)
1191                 write_fps.error |= (fflush(write_fps.cache_fp) != 0);
1192         if (write_fps.mark_fp)
1193                 write_fps.error |= (fflush(write_fps.mark_fp) != 0);
1194         if (write_fps.tags_fp)
1195                 write_fps.error |= (fflush(write_fps.tags_fp) != 0);
1196
1197         /* sync to filesystem */
1198         if (prefs_common.flush_metadata && write_fps.cache_fp)
1199                 write_fps.error |= (fsync(fileno(write_fps.cache_fp)) != 0);
1200         if (prefs_common.flush_metadata && write_fps.mark_fp)
1201                 write_fps.error |= (fsync(fileno(write_fps.mark_fp)) != 0);
1202         if (prefs_common.flush_metadata && write_fps.tags_fp)
1203                 write_fps.error |= (fsync(fileno(write_fps.tags_fp)) != 0);
1204
1205         /* close files */
1206         if (write_fps.cache_fp)
1207                 write_fps.error |= (fclose(write_fps.cache_fp) != 0);
1208         if (write_fps.mark_fp)
1209                 write_fps.error |= (fclose(write_fps.mark_fp) != 0);
1210         if (write_fps.tags_fp)
1211                 write_fps.error |= (fclose(write_fps.tags_fp) != 0);
1212
1213
1214         if (write_fps.error != 0) {
1215                 /* in case of error, forget all */
1216                 claws_unlink(new_cache);
1217                 claws_unlink(new_mark);
1218                 claws_unlink(new_tags);
1219                 g_free(new_cache);
1220                 g_free(new_mark);
1221                 g_free(new_tags);
1222                 return -1;
1223         } else {
1224                 /* switch files */
1225                 if (cache_file)
1226                         move_file(new_cache, cache_file, TRUE);
1227                 if (mark_file)
1228                         move_file(new_mark, mark_file, TRUE);
1229                 if (tags_file)
1230                         move_file(new_tags, tags_file, TRUE);
1231                 cache->last_access = time(NULL);
1232         }
1233
1234         g_free(new_cache);
1235         g_free(new_mark);
1236         g_free(new_tags);
1237         debug_print("done.\n");
1238         END_TIMING();
1239         return 0;
1240 }
1241