ba292132ea08f1773854810ea913dd3579f1a403
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2012 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  * 
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #include "claws-features.h"
23 #endif
24
25 #include "defs.h"
26
27 #define _GNU_SOURCE
28 #include <stdio.h>
29
30 #include <glib.h>
31 #include <glib/gi18n.h>
32 #ifdef _WIN32
33 # include <w32lib.h>
34 # define MAP_FAILED     ((char *) -1)
35 #else
36 # include <sys/mman.h>
37 #endif
38 #include <sys/types.h>
39 #include <sys/stat.h>
40
41 #include <time.h>
42
43 #include "msgcache.h"
44 #include "utils.h"
45 #include "procmsg.h"
46 #include "codeconv.h"
47 #include "timing.h"
48 #include "tags.h"
49 #include "prefs_common.h"
50
51 #ifdef HAVE_FWRITE_UNLOCKED
52 #define SC_FWRITE fwrite_unlocked
53 #else
54 #define SC_FWRITE fwrite
55 #endif
56
57 #if G_BYTE_ORDER == G_BIG_ENDIAN
58 #define bswap_32(x) \
59      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
60       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
61      
62 #define MMAP_TO_GUINT32(x)      \
63         (((x[3]&0xff)) |        \
64          ((x[2]&0xff) << 8) |   \
65          ((x[1]&0xff) << 16) |  \
66          ((x[0]&0xff) << 24))
67
68 #define MMAP_TO_GUINT32_SWAPPED(x)      \
69         (((x[0]&0xff)) |                \
70          ((x[1]&0xff) << 8) |           \
71          ((x[2]&0xff) << 16) |          \
72          ((x[3]&0xff) << 24))
73
74 static gboolean msgcache_use_mmap_read = TRUE;
75
76 #else
77 #define bswap_32(x) (x)
78
79 #define MMAP_TO_GUINT32(x)      \
80         (((x[0]&0xff)) |        \
81          ((x[1]&0xff) << 8) |   \
82          ((x[2]&0xff) << 16) |  \
83          ((x[3]&0xff) << 24))
84
85 #define MMAP_TO_GUINT32_SWAPPED(x)      \
86         (((x[0]&0xff)) |                \
87          ((x[1]&0xff) << 8) |           \
88          ((x[2]&0xff) << 16) |          \
89          ((x[3]&0xff) << 24))
90
91 static gboolean msgcache_use_mmap_read = TRUE;
92 #endif
93
94 static gboolean swapping = TRUE;
95
96 typedef enum
97 {
98         DATA_READ,
99         DATA_WRITE,
100         DATA_APPEND
101 } DataOpenMode;
102
103 struct _MsgCache {
104         GHashTable      *msgnum_table;
105         GHashTable      *msgid_table;
106         guint            memusage;
107         time_t           last_access;
108 };
109
110 typedef struct _StringConverter StringConverter;
111 struct _StringConverter {
112         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
113         void   (*free)    (StringConverter *converter);
114 };
115
116 typedef struct _StrdupConverter StrdupConverter;
117 struct _StrdupConverter {
118         StringConverter converter;
119 };
120
121 typedef struct _CharsetConverter CharsetConverter;
122 struct _CharsetConverter {
123         StringConverter converter;
124
125         gchar *srccharset;
126         gchar *dstcharset;
127 };
128
129 MsgCache *msgcache_new(void)
130 {
131         MsgCache *cache;
132         
133         cache = g_new0(MsgCache, 1),
134         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
135         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
136         cache->last_access = time(NULL);
137
138         return cache;
139 }
140
141 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
142 {
143         procmsg_msginfo_free((MsgInfo *)msginfo);
144         return TRUE;
145 }                                                                                         
146
147 void msgcache_destroy(MsgCache *cache)
148 {
149         cm_return_if_fail(cache != NULL);
150
151         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
152         g_hash_table_destroy(cache->msgid_table);
153         g_hash_table_destroy(cache->msgnum_table);
154         g_free(cache);
155 }
156
157 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
158 {
159         MsgInfo *newmsginfo;
160
161         cm_return_if_fail(cache != NULL);
162         cm_return_if_fail(msginfo != NULL);
163
164         newmsginfo = procmsg_msginfo_new_ref(msginfo);
165         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
166         if(newmsginfo->msgid != NULL)
167                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
168         cache->memusage += procmsg_msginfo_memusage(msginfo);
169         cache->last_access = time(NULL);
170
171         msginfo->folder->cache_dirty = TRUE;
172
173         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
174 }
175
176 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
177 {
178         MsgInfo *msginfo;
179
180         cm_return_if_fail(cache != NULL);
181
182         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
183         if(!msginfo)
184                 return;
185
186         cache->memusage -= procmsg_msginfo_memusage(msginfo);
187         if(msginfo->msgid)
188                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
189         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
190
191         msginfo->folder->cache_dirty = TRUE;
192
193         procmsg_msginfo_free(msginfo);
194         cache->last_access = time(NULL);
195
196
197         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
198 }
199
200 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
201 {
202         MsgInfo *oldmsginfo, *newmsginfo;
203         
204         cm_return_if_fail(cache != NULL);
205         cm_return_if_fail(msginfo != NULL);
206
207         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
208         if(oldmsginfo && oldmsginfo->msgid) 
209                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
210         if (oldmsginfo) {
211                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
212                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
213                 procmsg_msginfo_free(oldmsginfo);
214         }
215
216         newmsginfo = procmsg_msginfo_new_ref(msginfo);
217         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
218         if(newmsginfo->msgid)
219                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
220         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
221         cache->last_access = time(NULL);
222         
223         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
224
225         msginfo->folder->cache_dirty = TRUE;
226
227         return;
228 }
229
230 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
231 {
232         MsgInfo *msginfo;
233
234         cm_return_val_if_fail(cache != NULL, NULL);
235
236         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
237         if(!msginfo)
238                 return NULL;
239         cache->last_access = time(NULL);
240         
241         return procmsg_msginfo_new_ref(msginfo);
242 }
243
244 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
245 {
246         MsgInfo *msginfo;
247         
248         cm_return_val_if_fail(cache != NULL, NULL);
249         cm_return_val_if_fail(msgid != NULL, NULL);
250
251         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
252         if(!msginfo)
253                 return NULL;
254         cache->last_access = time(NULL);
255         
256         return procmsg_msginfo_new_ref(msginfo);        
257 }
258
259 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
260 {
261         MsgInfoList **listptr = user_data;
262         MsgInfo *msginfo = value;
263
264         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
265 }
266
267 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
268 {
269         MsgInfoList *msg_list = NULL;
270         START_TIMING("");
271         cm_return_val_if_fail(cache != NULL, NULL);
272
273         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
274         cache->last_access = time(NULL);
275         
276         msg_list = g_slist_reverse(msg_list);
277         END_TIMING();
278         return msg_list;
279 }
280
281 time_t msgcache_get_last_access_time(MsgCache *cache)
282 {
283         cm_return_val_if_fail(cache != NULL, 0);
284         
285         return cache->last_access;
286 }
287
288 gint msgcache_get_memory_usage(MsgCache *cache)
289 {
290         cm_return_val_if_fail(cache != NULL, 0);
291
292         return cache->memusage;
293 }
294
295 /*
296  *  Cache saving functions
297  */
298
299 #define READ_CACHE_DATA(data, fp, total_len) \
300 { \
301         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
302                 procmsg_msginfo_free(msginfo); \
303                 error = TRUE; \
304                 goto bail_err; \
305         } \
306         total_len += tmp_len; \
307 }
308
309 #define READ_CACHE_DATA_INT(n, fp) \
310 { \
311         guint32 idata; \
312         size_t ni; \
313  \
314         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
315                 g_warning("read_int: Cache data corrupted, read %zd of %zd at " \
316                           "offset %ld", ni, sizeof(idata), ftell(fp)); \
317                 procmsg_msginfo_free(msginfo); \
318                 error = TRUE; \
319                 goto bail_err; \
320         } else \
321                 n = swapping ? bswap_32(idata) : (idata);\
322 }
323
324 #define GET_CACHE_DATA_INT(n)                                                                   \
325 {                                                                                               \
326         if (rem_len < 4) {                                                                      \
327                 g_print("error at rem_len:%d\n", rem_len);                                      \
328                 error = TRUE;                                                                   \
329                 goto bail_err;                                                                  \
330         }                                                                                       \
331         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data)));     \
332         walk_data += 4; rem_len -= 4;                                                           \
333 }
334
335 #define GET_CACHE_DATA(data, total_len) \
336 { \
337         GET_CACHE_DATA_INT(tmp_len);    \
338         if (rem_len < tmp_len) {                                                                \
339                 g_print("error at rem_len:%d (tmp_len %d)\n", rem_len, tmp_len);                \
340                 error = TRUE;                                                                   \
341                 goto bail_err;                                                                  \
342         }                                                                                       \
343         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
344                 g_print("error at rem_len:%d\n", rem_len);\
345                 procmsg_msginfo_free(msginfo); \
346                 error = TRUE; \
347                 goto bail_err; \
348         } \
349         total_len += tmp_len; \
350         walk_data += tmp_len; rem_len -= tmp_len; \
351 }
352
353
354 #define WRITE_CACHE_DATA_INT(n, fp)                     \
355 {                                                       \
356         guint32 idata;                                  \
357                                                         \
358         idata = (guint32)bswap_32(n);                   \
359         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
360                 w_err = 1;                              \
361         wrote += 4;                                     \
362 }
363
364 #define PUT_CACHE_DATA_INT(n)                           \
365 {                                                       \
366         walk_data[0]=(((guint32)n)&0x000000ff);                 \
367         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
368         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
369         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
370         walk_data += 4;                                 \
371         wrote += 4;                                     \
372 }
373
374 #define WRITE_CACHE_DATA(data, fp) \
375 { \
376         size_t len;                                     \
377         if (data == NULL)                               \
378                 len = 0;                                \
379         else                                            \
380                 len = strlen(data);                     \
381         WRITE_CACHE_DATA_INT(len, fp);                  \
382         if (w_err == 0 && len > 0) {                    \
383                 if (SC_FWRITE(data, 1, len, fp) != len) \
384                         w_err = 1;                      \
385                 wrote += len;                           \
386         } \
387 }
388
389 #define PUT_CACHE_DATA(data)                            \
390 {                                                       \
391         size_t len;                                     \
392         if (data == NULL)                               \
393                 len = 0;                                \
394         else                                            \
395                 len = strlen(data);                     \
396         PUT_CACHE_DATA_INT(len);                        \
397         if (len > 0) {                                  \
398                 memcpy(walk_data, data, len);           \
399                 walk_data += len;                       \
400                 wrote += len;                           \
401         }                                               \
402 }
403
404 static FILE *msgcache_open_data_file(const gchar *file, guint version,
405                                      DataOpenMode mode,
406                                      gchar *buf, size_t buf_size)
407 {
408         FILE *fp;
409         gint32 data_ver;
410
411         cm_return_val_if_fail(file != NULL, NULL);
412
413         if (mode == DATA_WRITE) {
414                 int w_err = 0, wrote = 0;
415                 if ((fp = g_fopen(file, "wb")) == NULL) {
416                         FILE_OP_ERROR(file, "fopen");
417                         return NULL;
418                 }
419                 if (change_file_mode_rw(fp, file) < 0)
420                         FILE_OP_ERROR(file, "chmod");
421
422                 WRITE_CACHE_DATA_INT(version, fp);
423                 if (w_err != 0) {
424                         g_warning("failed to write int");
425                         fclose(fp);
426                         return NULL;
427                 }
428                 return fp;
429         }
430
431         /* check version */
432         if ((fp = g_fopen(file, "rb")) == NULL)
433                 debug_print("Mark/Cache file '%s' not found\n", file);
434         else {
435                 if (buf && buf_size > 0)
436                         setvbuf(fp, buf, _IOFBF, buf_size);
437                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
438                          version != bswap_32(data_ver)) {
439                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
440                                   file, bswap_32(data_ver), version);
441                         fclose(fp);
442                         fp = NULL;
443                 }
444                 data_ver = bswap_32(data_ver);
445         }
446         
447         if (mode == DATA_READ)
448                 return fp;
449
450         if (fp) {
451                 /* reopen with append mode */
452                 fclose(fp);
453                 if ((fp = g_fopen(file, "ab")) == NULL)
454                         FILE_OP_ERROR(file, "fopen");
455         } else {
456                 /* open with overwrite mode if mark file doesn't exist or
457                    version is different */
458                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
459                                             buf_size);
460         }
461
462         return fp;
463 }
464
465 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
466                                          StringConverter *conv)
467 {
468         gchar *tmpstr = NULL;
469         size_t ni;
470         guint32 len;
471
472         *str = NULL;
473         if (!swapping) {
474                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
475                     len > G_MAXINT) {
476                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
477                                   "of %zd bytes at offset %ld", ni, sizeof(len),
478                                   ftell(fp));
479                         return -1;
480                 }
481         } else {
482                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
483                     bswap_32(len) > G_MAXINT) {
484                         g_warning("read_data_str: Cache data (len) corrupted, read %zd "
485                                   "of %zd bytes at offset %ld", ni, sizeof(len),
486                                   ftell(fp));
487                         return -1;
488                 }
489                 len = bswap_32(len);
490         }
491
492         if (len == 0)
493                 return 0;
494
495         tmpstr = g_try_malloc(len + 1);
496
497         if(tmpstr == NULL) {
498                 return -1;
499         }
500
501         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
502                 g_warning("read_data_str: Cache data corrupted, read %zd of %u "
503                           "bytes at offset %ld",
504                           ni, len, ftell(fp));
505                 g_free(tmpstr);
506                 return -1;
507         }
508         tmpstr[len] = 0;
509
510         if (conv != NULL) {
511                 *str = conv->convert(conv, tmpstr);
512                 g_free(tmpstr);
513         } else 
514                 *str = tmpstr;
515
516         return len;
517 }
518
519 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
520                                          StringConverter *conv)
521 {
522         gchar *tmpstr = NULL;
523
524         *str = NULL;
525
526         if (len == 0)
527                 return 0;
528
529         if(len > 2*1024*1024) {
530                 g_warning("read_data_str: refusing to allocate %d bytes.", len);
531                 return -1;
532         }
533
534         tmpstr = g_try_malloc(len + 1);
535
536         if(tmpstr == NULL) {
537                 return -1;
538         }
539
540         memcpy(tmpstr, src, len);
541         tmpstr[len] = 0;
542
543         if (conv != NULL) {
544                 *str = conv->convert(conv, tmpstr);
545                 g_free(tmpstr);
546         } else 
547                 *str = tmpstr;
548
549         return len;
550 }
551
552 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
553 {
554         CharsetConverter *charsetconv = (CharsetConverter *) conv;
555
556         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
557 }
558
559 static void strconv_charset_free(StringConverter *conv)
560 {
561         CharsetConverter *charsetconv = (CharsetConverter *) conv;
562
563         g_free(charsetconv->srccharset);
564         g_free(charsetconv->dstcharset);
565 }
566
567 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
568 {
569         MsgCache *cache;
570         FILE *fp;
571         MsgInfo *msginfo;
572         MsgTmpFlags tmp_flags = 0;
573         gchar file_buf[BUFFSIZE];
574         guint32 num;
575         guint refnum;
576         gboolean error = FALSE;
577         StringConverter *conv = NULL;
578         gchar *srccharset = NULL;
579         const gchar *dstcharset = NULL;
580         gchar *ref = NULL;
581         guint memusage = 0;
582         gint tmp_len = 0, map_len = -1;
583         char *cache_data = NULL;
584         struct stat st;
585
586         cm_return_val_if_fail(cache_file != NULL, NULL);
587         cm_return_val_if_fail(item != NULL, NULL);
588
589         swapping = TRUE;
590
591         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
592          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
593          * it means it's the old version (not little-endian) on a big-endian machine. The code has
594          * no effect on x86 as their file doesn't change. */
595
596         if ((fp = msgcache_open_data_file
597                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
598                 if ((fp = msgcache_open_data_file
599                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
600                         return NULL;
601                 else
602                         swapping = FALSE;
603         }
604
605         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
606
607         if (folder_has_parent_of_type(item, F_QUEUE)) {
608                 tmp_flags |= MSG_QUEUED;
609         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
610                 tmp_flags |= MSG_DRAFT;
611         }
612
613         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0) {
614                 fclose(fp);
615                 return NULL;
616         }
617         dstcharset = CS_UTF_8;
618         if (srccharset == NULL || dstcharset == NULL) {
619                 conv = NULL;
620         } else if (strcmp(srccharset, dstcharset) == 0) {
621                 debug_print("using Noop Converter\n");
622
623                 conv = NULL;
624         } else {
625                 CharsetConverter *charsetconv;
626
627                 debug_print("using CharsetConverter\n");
628
629                 charsetconv = g_new0(CharsetConverter, 1);
630                 charsetconv->converter.convert = strconv_charset_convert;
631                 charsetconv->converter.free = strconv_charset_free;
632                 charsetconv->srccharset = g_strdup(srccharset);
633                 charsetconv->dstcharset = g_strdup(dstcharset);
634
635                 conv = (StringConverter *) charsetconv;
636         }
637         g_free(srccharset);
638
639         cache = msgcache_new();
640
641         if (msgcache_use_mmap_read == TRUE) {
642                 if (fstat(fileno(fp), &st) >= 0)
643                         map_len = st.st_size;
644                 else
645                         map_len = -1;
646                 if (map_len > 0) {
647 #ifdef G_OS_WIN32
648                         cache_data = NULL;
649                         HANDLE hFile, hMapping;
650                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
651                         if (hFile == (HANDLE) -1)
652                                 goto w32_fail;
653                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
654                         if (!hMapping)
655                                 goto w32_fail;
656                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
657                         CloseHandle (hMapping);
658                 w32_fail:
659                         ;
660 #else
661                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
662 #endif
663                 }
664         } else {
665                 cache_data = NULL;
666         }
667         if (cache_data != NULL && cache_data != MAP_FAILED) {
668                 int rem_len = map_len-ftell(fp);
669                 char *walk_data = cache_data+ftell(fp);
670
671                 while(rem_len > 0) {
672                         GET_CACHE_DATA_INT(num);
673                         
674                         msginfo = procmsg_msginfo_new();
675                         msginfo->msgnum = num;
676                         memusage += sizeof(MsgInfo);
677
678                         GET_CACHE_DATA_INT(msginfo->size);
679                         GET_CACHE_DATA_INT(msginfo->mtime);
680                         GET_CACHE_DATA_INT(msginfo->date_t);
681                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
682
683                         GET_CACHE_DATA(msginfo->fromname, memusage);
684
685                         GET_CACHE_DATA(msginfo->date, memusage);
686                         GET_CACHE_DATA(msginfo->from, memusage);
687                         GET_CACHE_DATA(msginfo->to, memusage);
688                         GET_CACHE_DATA(msginfo->cc, memusage);
689                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
690                         GET_CACHE_DATA(msginfo->subject, memusage);
691                         GET_CACHE_DATA(msginfo->msgid, memusage);
692                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
693                         GET_CACHE_DATA(msginfo->xref, memusage);
694
695                         GET_CACHE_DATA_INT(msginfo->planned_download);
696                         GET_CACHE_DATA_INT(msginfo->total_size);
697                         GET_CACHE_DATA_INT(refnum);
698
699                         for (; refnum != 0; refnum--) {
700                                 ref = NULL;
701
702                                 GET_CACHE_DATA(ref, memusage);
703
704                                 if (ref && *ref)
705                                         msginfo->references =
706                                                 g_slist_prepend(msginfo->references, ref);
707                         }
708                         if (msginfo->references)
709                                 msginfo->references =
710                                         g_slist_reverse(msginfo->references);
711
712                         msginfo->folder = item;
713                         msginfo->flags.tmp_flags |= tmp_flags;
714
715                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
716                         if(msginfo->msgid)
717                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
718                 }
719         } else {
720                 while (fread(&num, sizeof(num), 1, fp) == 1) {
721                         if (swapping)
722                                 num = bswap_32(num);
723
724                         msginfo = procmsg_msginfo_new();
725                         msginfo->msgnum = num;
726                         memusage += sizeof(MsgInfo);
727
728                         READ_CACHE_DATA_INT(msginfo->size, fp);
729                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
730                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
731                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
732
733                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
734
735                         READ_CACHE_DATA(msginfo->date, fp, memusage);
736                         READ_CACHE_DATA(msginfo->from, fp, memusage);
737                         READ_CACHE_DATA(msginfo->to, fp, memusage);
738                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
739                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
740                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
741                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
742                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
743                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
744
745                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
746                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
747                         READ_CACHE_DATA_INT(refnum, fp);
748
749                         for (; refnum != 0; refnum--) {
750                                 ref = NULL;
751
752                                 READ_CACHE_DATA(ref, fp, memusage);
753
754                                 if (ref && *ref)
755                                         msginfo->references =
756                                                 g_slist_prepend(msginfo->references, ref);
757                         }
758                         if (msginfo->references)
759                                 msginfo->references =
760                                         g_slist_reverse(msginfo->references);
761
762                         msginfo->folder = item;
763                         msginfo->flags.tmp_flags |= tmp_flags;
764
765                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
766                         if(msginfo->msgid)
767                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
768                 }
769         }
770 bail_err:
771         if (cache_data != NULL && cache_data != MAP_FAILED) {
772 #ifdef G_OS_WIN32
773                 UnmapViewOfFile((void*) cache_data);
774 #else
775                 munmap(cache_data, map_len);
776 #endif
777         }
778         fclose(fp);
779         if (conv != NULL) {
780                 if (conv->free != NULL)
781                         conv->free(conv);
782                 g_free(conv);
783         }
784
785         if(error) {
786                 msgcache_destroy(cache);
787                 return NULL;
788         }
789
790         cache->last_access = time(NULL);
791         cache->memusage = memusage;
792
793         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
794         debug_print("Cache size: %d messages, %u bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
795
796         return cache;
797 }
798
799 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
800 {
801         FILE *fp;
802         MsgInfo *msginfo;
803         MsgPermFlags perm_flags;
804         guint32 num;
805         gint map_len = -1;
806         char *cache_data = NULL;
807         struct stat st;
808         gboolean error = FALSE;
809
810         swapping = TRUE;
811
812         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
813          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
814          * it means it's the old version (not little-endian) on a big-endian machine. The code has
815          * no effect on x86 as their file doesn't change. */
816
817         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
818                 /* see if it isn't swapped ? */
819                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
820                         return;
821                 else
822                         swapping = FALSE; /* yay */
823         }
824         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
825         
826         if (msgcache_use_mmap_read) {
827                 if (fstat(fileno(fp), &st) >= 0)
828                         map_len = st.st_size;
829                 else
830                         map_len = -1;
831                 if (map_len > 0) {
832 #ifdef G_OS_WIN32
833                         cache_data = NULL;
834                         HANDLE hFile, hMapping;
835                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
836                         if (hFile == (HANDLE) -1)
837                                 goto w32_fail2;
838                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
839                         if (!hMapping)
840                                 goto w32_fail2;
841                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
842                         CloseHandle (hMapping);
843                 w32_fail2:
844                         ;
845 #else
846                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
847 #endif
848                 }
849         } else {
850                 cache_data = NULL;
851         }
852         if (cache_data != NULL && cache_data != MAP_FAILED) {
853                 int rem_len = map_len-ftell(fp);
854                 char *walk_data = cache_data+ftell(fp);
855
856                 while(rem_len > 0) {
857                         GET_CACHE_DATA_INT(num);
858                         GET_CACHE_DATA_INT(perm_flags);
859                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
860                         if(msginfo) {
861                                 msginfo->flags.perm_flags = perm_flags;
862                         }
863                 }
864         } else {
865                 while (fread(&num, sizeof(num), 1, fp) == 1) {
866                         if (swapping)
867                                 num = bswap_32(num);
868                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) {
869                                 error = TRUE;
870                                 break;
871                         }
872                         if (swapping)
873                                 perm_flags = bswap_32(perm_flags);
874                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
875                         if(msginfo) {
876                                 msginfo->flags.perm_flags = perm_flags;
877                         }
878                 }       
879         }
880 bail_err:
881         if (cache_data != NULL && cache_data != MAP_FAILED) {
882 #ifdef G_OS_WIN32
883                 UnmapViewOfFile((void*) cache_data);
884 #else
885                 munmap(cache_data, map_len);
886 #endif
887         }
888         fclose(fp);
889         if (error) {
890                 debug_print("error reading cache mark from %s\n", mark_file);
891         }
892 }
893
894 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
895 {
896         FILE *fp;
897         MsgInfo *msginfo;
898         guint32 num;
899         gint map_len = -1;
900         char *cache_data = NULL;
901         struct stat st;
902         gboolean error = FALSE;
903
904         swapping = TRUE;
905
906         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
907          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
908          * it means it's the old version (not little-endian) on a big-endian machine. The code has
909          * no effect on x86 as their file doesn't change. */
910
911         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
912                 /* see if it isn't swapped ? */
913                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
914                         return;
915                 else
916                         swapping = FALSE; /* yay */
917         }
918         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
919         
920         if (msgcache_use_mmap_read) {
921                 if (fstat(fileno(fp), &st) >= 0)
922                         map_len = st.st_size;
923                 else
924                         map_len = -1;
925                 if (map_len > 0) {
926 #ifdef G_OS_WIN32
927                         cache_data = NULL;
928                         HANDLE hFile, hMapping;
929                         hFile = (HANDLE) _get_osfhandle (fileno(fp));
930                         if (hFile == (HANDLE) -1)
931                                 goto w32_fail6;
932                         hMapping = CreateFileMapping(hFile, NULL, PAGE_WRITECOPY, 0, 0, NULL);
933                         if (!hMapping)
934                                 goto w32_fail6;
935                         cache_data = (unsigned char *)MapViewOfFile(hMapping, FILE_MAP_COPY, 0, 0, 0);
936                         CloseHandle (hMapping);
937                 w32_fail6:
938                         ;
939 #else
940                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
941 #endif
942                 }
943         } else {
944                 cache_data = NULL;
945         }
946         if (cache_data != NULL && cache_data != MAP_FAILED) {
947                 int rem_len = map_len-ftell(fp);
948                 char *walk_data = cache_data+ftell(fp);
949
950                 while(rem_len > 0) {
951                         gint id = -1;
952                         GET_CACHE_DATA_INT(num);
953                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
954                         if(msginfo) {
955                                 g_slist_free(msginfo->tags);
956                                 msginfo->tags = NULL;
957                                 do {
958                                         GET_CACHE_DATA_INT(id);
959                                         if (id > 0) {
960                                                 msginfo->tags = g_slist_prepend(
961                                                         msginfo->tags, 
962                                                         GINT_TO_POINTER(id));
963                                         }
964                                 } while (id > 0);
965                                 msginfo->tags = g_slist_reverse(msginfo->tags);
966                         }
967                 }
968         } else {
969                 while (fread(&num, sizeof(num), 1, fp) == 1) {
970                         gint id = -1;
971                         if (swapping)
972                                 num = bswap_32(num);
973                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
974                         if(msginfo) {
975                                 g_slist_free(msginfo->tags);
976                                 msginfo->tags = NULL;
977                                 do {
978                                         if (fread(&id, sizeof(id), 1, fp) != 1) 
979                                                 id = -1;
980                                         if (swapping)
981                                                 id = bswap_32(id);
982                                         if (id > 0) {
983                                                 msginfo->tags = g_slist_prepend(
984                                                         msginfo->tags, 
985                                                         GINT_TO_POINTER(id));
986                                         }
987                                 } while (id > 0);
988                                 msginfo->tags = g_slist_reverse(msginfo->tags);
989                         }
990                 }
991         }
992 bail_err:
993         if (cache_data != NULL && cache_data != MAP_FAILED) {
994 #ifdef G_OS_WIN32
995                 UnmapViewOfFile((void*) cache_data);
996 #else
997                 munmap(cache_data, map_len);
998 #endif
999         }
1000         fclose(fp);
1001         if (error) {
1002                 debug_print("error reading cache tags from %s\n", tags_file);
1003         }
1004 }
1005
1006 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
1007 {
1008         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
1009         GSList *cur;
1010         int w_err = 0, wrote = 0;
1011
1012         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1013         WRITE_CACHE_DATA_INT(msginfo->size, fp);
1014         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
1015         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
1016         WRITE_CACHE_DATA_INT(flags, fp);
1017
1018         WRITE_CACHE_DATA(msginfo->fromname, fp);
1019
1020         WRITE_CACHE_DATA(msginfo->date, fp);
1021         WRITE_CACHE_DATA(msginfo->from, fp);
1022         WRITE_CACHE_DATA(msginfo->to, fp);
1023         WRITE_CACHE_DATA(msginfo->cc, fp);
1024         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
1025         WRITE_CACHE_DATA(msginfo->subject, fp);
1026         WRITE_CACHE_DATA(msginfo->msgid, fp);
1027         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
1028         WRITE_CACHE_DATA(msginfo->xref, fp);
1029         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
1030         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
1031         
1032         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
1033
1034         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
1035                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
1036         }
1037         return w_err ? -1 : wrote;
1038 }
1039
1040 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
1041 {
1042         MsgPermFlags flags = msginfo->flags.perm_flags;
1043         int w_err = 0, wrote = 0;
1044         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1045         WRITE_CACHE_DATA_INT(flags, fp);
1046         return w_err ? -1 : wrote;
1047 }
1048
1049 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
1050 {
1051         GSList *cur = msginfo->tags;
1052         int w_err = 0, wrote = 0;
1053
1054         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
1055         for (; cur; cur = cur->next) {
1056                 gint id = GPOINTER_TO_INT(cur->data);
1057                 if (tags_get_tag(id) != NULL) {
1058                         WRITE_CACHE_DATA_INT(id, fp);
1059                 }
1060         }
1061         WRITE_CACHE_DATA_INT(-1, fp);
1062
1063         return w_err ? -1 : wrote;
1064 }
1065
1066 struct write_fps
1067 {
1068         FILE *cache_fp;
1069         FILE *mark_fp;
1070         FILE *tags_fp;
1071         int error;
1072         guint cache_size;
1073         guint mark_size;
1074         guint tags_size;
1075 };
1076
1077 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1078 {
1079         MsgInfo *msginfo;
1080         struct write_fps *write_fps;
1081         int tmp;
1082
1083         msginfo = (MsgInfo *)value;
1084         write_fps = user_data;
1085
1086         if (write_fps->cache_fp) {
1087                 tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1088                 if (tmp < 0)
1089                         write_fps->error = 1;
1090                 else
1091                         write_fps->cache_size += tmp;
1092         }
1093         if (write_fps->mark_fp) {
1094         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1095                 if (tmp < 0)
1096                         write_fps->error = 1;
1097                 else
1098                         write_fps->mark_size += tmp;
1099                 }
1100         if (write_fps->tags_fp) {
1101                 tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1102                 if (tmp < 0)
1103                         write_fps->error = 1;
1104                 else
1105                         write_fps->tags_size += tmp;
1106         }
1107 }
1108
1109 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1110 {
1111         struct write_fps write_fps;
1112         gchar *new_cache, *new_mark, *new_tags;
1113         int w_err = 0, wrote = 0;
1114
1115         START_TIMING("");
1116         cm_return_val_if_fail(cache != NULL, -1);
1117
1118         new_cache = g_strconcat(cache_file, ".new", NULL);
1119         new_mark  = g_strconcat(mark_file, ".new", NULL);
1120         new_tags  = g_strconcat(tags_file, ".new", NULL);
1121
1122         write_fps.error = 0;
1123         write_fps.cache_size = 0;
1124         write_fps.mark_size = 0;
1125         write_fps.tags_size = 0;
1126
1127         /* open files and write headers */
1128
1129         if (cache_file) {
1130                 write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1131                         DATA_WRITE, NULL, 0);
1132                 if (write_fps.cache_fp == NULL) {
1133                         g_free(new_cache);
1134                         g_free(new_mark);
1135                         g_free(new_tags);
1136                         return -1;
1137                 }
1138                 WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1139         } else {
1140                 write_fps.cache_fp = NULL;
1141         }
1142
1143         if (w_err != 0) {
1144                 g_warning("failed to write charset");
1145                 if (write_fps.cache_fp)
1146                         fclose(write_fps.cache_fp);
1147                 claws_unlink(new_cache);
1148                 g_free(new_cache);
1149                 g_free(new_mark);
1150                 g_free(new_tags);
1151                 return -1;
1152         }
1153
1154         if (mark_file) {
1155                 write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1156                         DATA_WRITE, NULL, 0);
1157                 if (write_fps.mark_fp == NULL) {
1158                         if (write_fps.cache_fp)
1159                                 fclose(write_fps.cache_fp);
1160                         claws_unlink(new_cache);
1161                         g_free(new_cache);
1162                         g_free(new_mark);
1163                         g_free(new_tags);
1164                         return -1;
1165                 }
1166         } else {
1167                 write_fps.mark_fp = NULL;
1168         }
1169
1170         if (tags_file) {
1171                 write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1172                         DATA_WRITE, NULL, 0);
1173                 if (write_fps.tags_fp == NULL) {
1174                         if (write_fps.cache_fp)
1175                                 fclose(write_fps.cache_fp);
1176                         if (write_fps.mark_fp)
1177                                 fclose(write_fps.mark_fp);
1178                         claws_unlink(new_cache);
1179                         claws_unlink(new_mark);
1180                         g_free(new_cache);
1181                         g_free(new_mark);
1182                         g_free(new_tags);
1183                         return -1;
1184                 }
1185         } else {
1186                 write_fps.tags_fp = NULL;
1187         }
1188
1189         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1190
1191         if (write_fps.cache_fp && change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1192                 FILE_OP_ERROR(new_cache, "chmod");
1193
1194         /* headers written, note file size */
1195         if (write_fps.cache_fp)
1196                 write_fps.cache_size = ftell(write_fps.cache_fp);
1197         if (write_fps.mark_fp)
1198                 write_fps.mark_size = ftell(write_fps.mark_fp);
1199         if (write_fps.tags_fp)
1200                 write_fps.tags_size = ftell(write_fps.tags_fp);
1201
1202 #ifdef HAVE_FWRITE_UNLOCKED
1203         /* lock files for write once (instead of once per fwrite) */
1204         if (write_fps.cache_fp)
1205                 flockfile(write_fps.cache_fp);
1206         if (write_fps.mark_fp)
1207                 flockfile(write_fps.mark_fp);
1208         if (write_fps.tags_fp)
1209                 flockfile(write_fps.tags_fp);
1210 #endif
1211         /* write data to the files */
1212         g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1213 #ifdef HAVE_FWRITE_UNLOCKED
1214         /* unlock files */
1215         if (write_fps.cache_fp)
1216                 funlockfile(write_fps.cache_fp);
1217         if (write_fps.mark_fp)
1218                 funlockfile(write_fps.mark_fp);
1219         if (write_fps.tags_fp)
1220                 funlockfile(write_fps.tags_fp);
1221 #endif
1222         /* flush buffers */
1223         if (write_fps.cache_fp)
1224                 write_fps.error |= (fflush(write_fps.cache_fp) != 0);
1225         if (write_fps.mark_fp)
1226                 write_fps.error |= (fflush(write_fps.mark_fp) != 0);
1227         if (write_fps.tags_fp)
1228                 write_fps.error |= (fflush(write_fps.tags_fp) != 0);
1229
1230         /* sync to filesystem */
1231         if (prefs_common.flush_metadata && write_fps.cache_fp)
1232                 write_fps.error |= (fsync(fileno(write_fps.cache_fp)) != 0);
1233         if (prefs_common.flush_metadata && write_fps.mark_fp)
1234                 write_fps.error |= (fsync(fileno(write_fps.mark_fp)) != 0);
1235         if (prefs_common.flush_metadata && write_fps.tags_fp)
1236                 write_fps.error |= (fsync(fileno(write_fps.tags_fp)) != 0);
1237
1238         /* close files */
1239         if (write_fps.cache_fp)
1240                 write_fps.error |= (fclose(write_fps.cache_fp) != 0);
1241         if (write_fps.mark_fp)
1242                 write_fps.error |= (fclose(write_fps.mark_fp) != 0);
1243         if (write_fps.tags_fp)
1244                 write_fps.error |= (fclose(write_fps.tags_fp) != 0);
1245
1246
1247         if (write_fps.error != 0) {
1248                 /* in case of error, forget all */
1249                 claws_unlink(new_cache);
1250                 claws_unlink(new_mark);
1251                 claws_unlink(new_tags);
1252                 g_free(new_cache);
1253                 g_free(new_mark);
1254                 g_free(new_tags);
1255                 return -1;
1256         } else {
1257                 /* switch files */
1258                 if (cache_file)
1259                         move_file(new_cache, cache_file, TRUE);
1260                 if (mark_file)
1261                         move_file(new_mark, mark_file, TRUE);
1262                 if (tags_file)
1263                         move_file(new_tags, tags_file, TRUE);
1264                 cache->last_access = time(NULL);
1265         }
1266
1267         g_free(new_cache);
1268         g_free(new_mark);
1269         g_free(new_tags);
1270         debug_print("done.\n");
1271         END_TIMING();
1272         return 0;
1273 }
1274