2007-01-05 [paul] 2.6.1cvs110
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2007 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #endif
23
24 #include "defs.h"
25
26 #define _GNU_SOURCE
27 #include <stdio.h>
28
29 #include <glib.h>
30 #include <glib/gi18n.h>
31 #include <sys/mman.h>
32 #include <sys/types.h>
33 #include <sys/stat.h>
34
35 #include <time.h>
36
37 #include "msgcache.h"
38 #include "utils.h"
39 #include "procmsg.h"
40 #include "codeconv.h"
41 #include "timing.h"
42
43 #ifdef HAVE_FWRITE_UNLOCKED
44 #define SC_FWRITE fwrite_unlocked
45 #else
46 #define SC_FWRITE fwrite
47 #endif
48
49 #if G_BYTE_ORDER == G_BIG_ENDIAN
50 #define bswap_32(x) \
51      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
52       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
53      
54 #define MMAP_TO_GUINT32(x)      \
55         (((x[3]&0xff)) |        \
56          ((x[2]&0xff) << 8) |   \
57          ((x[1]&0xff) << 16) |  \
58          ((x[0]&0xff) << 24))
59
60 #define MMAP_TO_GUINT32_SWAPPED(x)      \
61         (((x[0]&0xff)) |                \
62          ((x[1]&0xff) << 8) |           \
63          ((x[2]&0xff) << 16) |          \
64          ((x[3]&0xff) << 24))
65
66 static gboolean msgcache_use_mmap_read = TRUE;
67 static gboolean msgcache_use_mmap_write = FALSE;
68
69 #else
70 #define bswap_32(x) (x)
71
72 #define MMAP_TO_GUINT32(x)      \
73         (((x[0]&0xff)) |        \
74          ((x[1]&0xff) << 8) |   \
75          ((x[2]&0xff) << 16) |  \
76          ((x[3]&0xff) << 24))
77
78 #define MMAP_TO_GUINT32_SWAPPED(x)      \
79         (((x[0]&0xff)) |                \
80          ((x[1]&0xff) << 8) |           \
81          ((x[2]&0xff) << 16) |          \
82          ((x[3]&0xff) << 24))
83
84 static gboolean msgcache_use_mmap_read = TRUE;
85 static gboolean msgcache_use_mmap_write = FALSE;
86 #endif
87
88 static gboolean swapping = TRUE;
89
90 typedef enum
91 {
92         DATA_READ,
93         DATA_WRITE,
94         DATA_APPEND
95 } DataOpenMode;
96
97 struct _MsgCache {
98         GHashTable      *msgnum_table;
99         GHashTable      *msgid_table;
100         guint            memusage;
101         time_t           last_access;
102 };
103
104 typedef struct _StringConverter StringConverter;
105 struct _StringConverter {
106         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
107         void   (*free)    (StringConverter *converter);
108 };
109
110 typedef struct _StrdupConverter StrdupConverter;
111 struct _StrdupConverter {
112         StringConverter converter;
113 };
114
115 typedef struct _CharsetConverter CharsetConverter;
116 struct _CharsetConverter {
117         StringConverter converter;
118
119         gchar *srccharset;
120         gchar *dstcharset;
121 };
122
123 MsgCache *msgcache_new(void)
124 {
125         MsgCache *cache;
126         
127         cache = g_new0(MsgCache, 1),
128         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
129         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
130         cache->last_access = time(NULL);
131
132         return cache;
133 }
134
135 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
136 {
137         procmsg_msginfo_free((MsgInfo *)msginfo);
138         return TRUE;
139 }                                                                                         
140
141 void msgcache_destroy(MsgCache *cache)
142 {
143         g_return_if_fail(cache != NULL);
144
145         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
146         g_hash_table_destroy(cache->msgid_table);
147         g_hash_table_destroy(cache->msgnum_table);
148         g_free(cache);
149 }
150
151 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
152 {
153         MsgInfo *newmsginfo;
154
155         g_return_if_fail(cache != NULL);
156         g_return_if_fail(msginfo != NULL);
157
158         newmsginfo = procmsg_msginfo_new_ref(msginfo);
159         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
160         if(newmsginfo->msgid != NULL)
161                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
162         cache->memusage += procmsg_msginfo_memusage(msginfo);
163         cache->last_access = time(NULL);
164
165         debug_print("Cache size: %d messages, %d bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
166 }
167
168 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
169 {
170         MsgInfo *msginfo;
171
172         g_return_if_fail(cache != NULL);
173         g_return_if_fail(msgnum > 0);
174
175         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
176         if(!msginfo)
177                 return;
178
179         cache->memusage -= procmsg_msginfo_memusage(msginfo);
180         if(msginfo->msgid)
181                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
182         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
183         procmsg_msginfo_free(msginfo);
184         cache->last_access = time(NULL);
185
186         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
187 }
188
189 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
190 {
191         MsgInfo *oldmsginfo, *newmsginfo;
192         
193         g_return_if_fail(cache != NULL);
194         g_return_if_fail(msginfo != NULL);
195
196         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
197         if(oldmsginfo && oldmsginfo->msgid) 
198                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
199         if (oldmsginfo) {
200                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
201                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
202                 procmsg_msginfo_free(oldmsginfo);
203         }
204
205         newmsginfo = procmsg_msginfo_new_ref(msginfo);
206         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
207         if(newmsginfo->msgid)
208                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
209         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
210         cache->last_access = time(NULL);
211         
212         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
213
214         return;
215 }
216
217 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
218 {
219         MsgInfo *msginfo;
220
221         g_return_val_if_fail(cache != NULL, NULL);
222
223         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
224         if(!msginfo)
225                 return NULL;
226         cache->last_access = time(NULL);
227         
228         return procmsg_msginfo_new_ref(msginfo);
229 }
230
231 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
232 {
233         MsgInfo *msginfo;
234         
235         g_return_val_if_fail(cache != NULL, NULL);
236         g_return_val_if_fail(msgid != NULL, NULL);
237
238         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
239         if(!msginfo)
240                 return NULL;
241         cache->last_access = time(NULL);
242         
243         return procmsg_msginfo_new_ref(msginfo);        
244 }
245
246 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
247 {
248         MsgInfoList **listptr = user_data;
249         MsgInfo *msginfo = value;
250
251         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
252 }
253
254 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
255 {
256         MsgInfoList *msg_list = NULL;
257         START_TIMING("");
258         g_return_val_if_fail(cache != NULL, NULL);
259
260         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
261         cache->last_access = time(NULL);
262         
263         msg_list = g_slist_reverse(msg_list);
264         END_TIMING();
265         return msg_list;
266 }
267
268 time_t msgcache_get_last_access_time(MsgCache *cache)
269 {
270         g_return_val_if_fail(cache != NULL, 0);
271         
272         return cache->last_access;
273 }
274
275 gint msgcache_get_memory_usage(MsgCache *cache)
276 {
277         g_return_val_if_fail(cache != NULL, 0);
278
279         return cache->memusage;
280 }
281
282 /*
283  *  Cache saving functions
284  */
285
286 #define READ_CACHE_DATA(data, fp, total_len) \
287 { \
288         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
289                 procmsg_msginfo_free(msginfo); \
290                 error = TRUE; \
291                 goto bail_err; \
292         } \
293         total_len += tmp_len; \
294 }
295
296 #define READ_CACHE_DATA_INT(n, fp) \
297 { \
298         guint32 idata; \
299         size_t ni; \
300  \
301         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
302                 g_warning("read_int: Cache data corrupted, read %d of %d at " \
303                           "offset %ld\n", ni, sizeof(idata), ftell(fp)); \
304                 procmsg_msginfo_free(msginfo); \
305                 error = TRUE; \
306                 goto bail_err; \
307         } else \
308                 n = swapping ? bswap_32(idata) : (idata);\
309 }
310
311 #define GET_CACHE_DATA_INT(n) \
312 { \
313         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data))); \
314         walk_data += 4; rem_len -= 4;                   \
315 }
316
317 #define GET_CACHE_DATA(data, total_len) \
318 { \
319         GET_CACHE_DATA_INT(tmp_len);    \
320         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
321                 printf("error at rem_len:%d\n", rem_len);\
322                 procmsg_msginfo_free(msginfo); \
323                 error = TRUE; \
324                 goto bail_err; \
325         } \
326         total_len += tmp_len; \
327         walk_data += tmp_len; rem_len -= tmp_len; \
328 }
329
330
331 #define WRITE_CACHE_DATA_INT(n, fp)                     \
332 {                                                       \
333         guint32 idata;                                  \
334                                                         \
335         idata = (guint32)bswap_32(n);                   \
336         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
337                 w_err = 1;                              \
338         wrote += 4;                                     \
339 }
340
341 #define PUT_CACHE_DATA_INT(n)                           \
342 {                                                       \
343         walk_data[0]=(((guint32)n)&0x000000ff);                 \
344         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
345         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
346         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
347         walk_data += 4;                                 \
348         wrote += 4;                                     \
349 }
350
351 #define WRITE_CACHE_DATA(data, fp) \
352 { \
353         size_t len;                                     \
354         if (data == NULL)                               \
355                 len = 0;                                \
356         else                                            \
357                 len = strlen(data);                     \
358         WRITE_CACHE_DATA_INT(len, fp);                  \
359         if (w_err == 0 && len > 0) {                    \
360                 if (SC_FWRITE(data, 1, len, fp) != len) \
361                         w_err = 1;                      \
362                 wrote += len;                           \
363         } \
364 }
365
366 #define PUT_CACHE_DATA(data)                            \
367 {                                                       \
368         size_t len;                                     \
369         if (data == NULL)                               \
370                 len = 0;                                \
371         else                                            \
372                 len = strlen(data);                     \
373         PUT_CACHE_DATA_INT(len);                        \
374         if (len > 0) {                                  \
375                 memcpy(walk_data, data, len);           \
376                 walk_data += len;                       \
377                 wrote += len;                           \
378         }                                               \
379 }
380
381 static FILE *msgcache_open_data_file(const gchar *file, guint version,
382                                      DataOpenMode mode,
383                                      gchar *buf, size_t buf_size)
384 {
385         FILE *fp;
386         gint32 data_ver;
387
388         g_return_val_if_fail(file != NULL, NULL);
389
390         if (mode == DATA_WRITE) {
391                 int w_err = 0, wrote = 0;
392                 if ((fp = g_fopen(file, "w+")) == NULL) {
393                         FILE_OP_ERROR(file, "fopen");
394                         return NULL;
395                 }
396                 if (change_file_mode_rw(fp, file) < 0)
397                         FILE_OP_ERROR(file, "chmod");
398
399                 WRITE_CACHE_DATA_INT(version, fp);
400                 if (w_err != 0) {
401                         g_warning("failed to write int\n");
402                         fclose(fp);
403                         return NULL;
404                 }
405                 return fp;
406         }
407
408         /* check version */
409         if ((fp = g_fopen(file, "rb")) == NULL)
410                 debug_print("Mark/Cache file '%s' not found\n", file);
411         else {
412                 if (buf && buf_size > 0)
413                         setvbuf(fp, buf, _IOFBF, buf_size);
414                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
415                          version != bswap_32(data_ver)) {
416                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
417                                   file, bswap_32(data_ver), version);
418                         fclose(fp);
419                         fp = NULL;
420                 }
421                 data_ver = bswap_32(data_ver);
422         }
423         
424         if (mode == DATA_READ)
425                 return fp;
426
427         if (fp) {
428                 /* reopen with append mode */
429                 fclose(fp);
430                 if ((fp = g_fopen(file, "ab")) == NULL)
431                         FILE_OP_ERROR(file, "fopen");
432         } else {
433                 /* open with overwrite mode if mark file doesn't exist or
434                    version is different */
435                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
436                                             buf_size);
437         }
438
439         return fp;
440 }
441
442 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
443                                          StringConverter *conv)
444 {
445         gchar *tmpstr = NULL;
446         size_t ni;
447         guint32 len;
448
449         *str = NULL;
450         if (!swapping) {
451                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
452                     len > G_MAXINT) {
453                         g_warning("read_data_str: Cache data (len) corrupted, read %d "
454                                   "of %d bytes at offset %ld\n", ni, sizeof(len), 
455                                   ftell(fp));
456                         return -1;
457                 }
458         } else {
459                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
460                     bswap_32(len) > G_MAXINT) {
461                         g_warning("read_data_str: Cache data (len) corrupted, read %d "
462                                   "of %d bytes at offset %ld\n", ni, sizeof(len), 
463                                   ftell(fp));
464                         return -1;
465                 }
466                 len = bswap_32(len);
467         }
468
469         if (len == 0)
470                 return 0;
471
472         tmpstr = g_try_malloc(len + 1);
473
474         if(tmpstr == NULL) {
475                 g_warning("read_data_str: can't g_malloc %d bytes - cache data probably corrupted.\n", len);
476                 return -1;
477         }
478
479         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
480                 g_warning("read_data_str: Cache data corrupted, read %d of %d "
481                           "bytes at offset %ld\n", 
482                           ni, len, ftell(fp));
483                 g_free(tmpstr);
484                 return -1;
485         }
486         tmpstr[len] = 0;
487
488         if (conv != NULL) {
489                 *str = conv->convert(conv, tmpstr);
490                 g_free(tmpstr);
491         } else 
492                 *str = tmpstr;
493
494         return len;
495 }
496
497 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
498                                          StringConverter *conv)
499 {
500         gchar *tmpstr = NULL;
501
502         *str = NULL;
503
504         if (len == 0)
505                 return 0;
506
507         if(len > 2*1024*1024) {
508                 g_warning("read_data_str: refusing to allocate %d bytes.\n", len);
509                 return -1;
510         }
511
512         tmpstr = g_try_malloc(len + 1);
513
514         if(tmpstr == NULL) {
515                 g_warning("read_data_str: can't g_malloc %d bytes - cache data probably corrupted.\n", len);
516                 return -1;
517         }
518
519         strncpy(tmpstr, src, len);
520
521         tmpstr[len] = 0;
522
523         if (conv != NULL) {
524                 *str = conv->convert(conv, tmpstr);
525                 g_free(tmpstr);
526         } else 
527                 *str = tmpstr;
528
529         return len;
530 }
531
532 gchar *strconv_strdup_convert(StringConverter *conv, gchar *srcstr)
533 {
534         return g_strdup(srcstr);
535 }
536
537 gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
538 {
539         CharsetConverter *charsetconv = (CharsetConverter *) conv;
540
541         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
542 }
543
544 void strconv_charset_free(StringConverter *conv)
545 {
546         CharsetConverter *charsetconv = (CharsetConverter *) conv;
547
548         g_free(charsetconv->srccharset);
549         g_free(charsetconv->dstcharset);
550 }
551
552 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
553 {
554         MsgCache *cache;
555         FILE *fp;
556         MsgInfo *msginfo;
557         MsgTmpFlags tmp_flags = 0;
558         gchar file_buf[BUFFSIZE];
559         guint32 num;
560         guint refnum;
561         gboolean error = FALSE;
562         StringConverter *conv = NULL;
563         gchar *srccharset = NULL;
564         const gchar *dstcharset = NULL;
565         gchar *ref = NULL;
566         guint memusage = 0;
567         gint tmp_len = 0, map_len = -1;
568         char *cache_data = NULL;
569         struct stat st;
570
571         g_return_val_if_fail(cache_file != NULL, NULL);
572         g_return_val_if_fail(item != NULL, NULL);
573
574         swapping = TRUE;
575
576         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
577          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
578          * it means it's the old version (not little-endian) on a big-endian machine. The code has
579          * no effect on x86 as their file doesn't change. */
580
581         if ((fp = msgcache_open_data_file
582                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
583                 if ((fp = msgcache_open_data_file
584                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
585                         return NULL;
586                 else
587                         swapping = FALSE;
588         }
589
590         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
591
592         if (folder_has_parent_of_type(item, F_QUEUE)) {
593                 tmp_flags |= MSG_QUEUED;
594         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
595                 tmp_flags |= MSG_DRAFT;
596         }
597
598         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0)
599                 return NULL;
600         dstcharset = CS_UTF_8;
601         if (srccharset == NULL || dstcharset == NULL) {
602                 conv = NULL;
603         } else if (strcmp(srccharset, dstcharset) == 0) {
604                 debug_print("using Noop Converter\n");
605
606                 conv = NULL;
607         } else {
608                 CharsetConverter *charsetconv;
609
610                 debug_print("using CharsetConverter\n");
611
612                 charsetconv = g_new0(CharsetConverter, 1);
613                 charsetconv->converter.convert = strconv_charset_convert;
614                 charsetconv->converter.free = strconv_charset_free;
615                 charsetconv->srccharset = g_strdup(srccharset);
616                 charsetconv->dstcharset = g_strdup(dstcharset);
617
618                 conv = (StringConverter *) charsetconv;
619         }
620         g_free(srccharset);
621
622         cache = msgcache_new();
623
624         if (msgcache_use_mmap_read == TRUE) {
625                 if (fstat(fileno(fp), &st) >= 0)
626                         map_len = st.st_size;
627                 else
628                         map_len = -1;
629                 if (map_len > 0)
630                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
631         } else {
632                 cache_data = NULL;
633         }
634         if (cache_data != NULL && cache_data != MAP_FAILED) {
635                 int rem_len = map_len-ftell(fp);
636                 char *walk_data = cache_data+ftell(fp);
637
638                 while(rem_len > 0) {
639                         GET_CACHE_DATA_INT(num);
640                         
641                         msginfo = procmsg_msginfo_new();
642                         msginfo->msgnum = num;
643                         memusage += sizeof(MsgInfo);
644
645                         GET_CACHE_DATA_INT(msginfo->size);
646                         GET_CACHE_DATA_INT(msginfo->mtime);
647                         GET_CACHE_DATA_INT(msginfo->date_t);
648                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
649
650                         GET_CACHE_DATA(msginfo->fromname, memusage);
651
652                         GET_CACHE_DATA(msginfo->date, memusage);
653                         GET_CACHE_DATA(msginfo->from, memusage);
654                         GET_CACHE_DATA(msginfo->to, memusage);
655                         GET_CACHE_DATA(msginfo->cc, memusage);
656                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
657                         GET_CACHE_DATA(msginfo->subject, memusage);
658                         GET_CACHE_DATA(msginfo->msgid, memusage);
659                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
660                         GET_CACHE_DATA(msginfo->xref, memusage);
661
662                         GET_CACHE_DATA_INT(msginfo->planned_download);
663                         GET_CACHE_DATA_INT(msginfo->total_size);
664                         GET_CACHE_DATA_INT(refnum);
665
666                         for (; refnum != 0; refnum--) {
667                                 ref = NULL;
668
669                                 GET_CACHE_DATA(ref, memusage);
670
671                                 if (ref && *ref)
672                                         msginfo->references =
673                                                 g_slist_prepend(msginfo->references, ref);
674                         }
675                         if (msginfo->references)
676                                 msginfo->references =
677                                         g_slist_reverse(msginfo->references);
678
679                         msginfo->folder = item;
680                         msginfo->flags.tmp_flags |= tmp_flags;
681
682                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
683                         if(msginfo->msgid)
684                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
685                 }
686                 
687                 munmap(cache_data, map_len);
688         } else {
689                 while (fread(&num, sizeof(num), 1, fp) == 1) {
690                         if (swapping)
691                                 num = bswap_32(num);
692
693                         msginfo = procmsg_msginfo_new();
694                         msginfo->msgnum = num;
695                         memusage += sizeof(MsgInfo);
696
697                         READ_CACHE_DATA_INT(msginfo->size, fp);
698                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
699                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
700                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
701
702                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
703
704                         READ_CACHE_DATA(msginfo->date, fp, memusage);
705                         READ_CACHE_DATA(msginfo->from, fp, memusage);
706                         READ_CACHE_DATA(msginfo->to, fp, memusage);
707                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
708                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
709                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
710                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
711                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
712                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
713
714                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
715                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
716                         READ_CACHE_DATA_INT(refnum, fp);
717
718                         for (; refnum != 0; refnum--) {
719                                 ref = NULL;
720
721                                 READ_CACHE_DATA(ref, fp, memusage);
722
723                                 if (ref && *ref)
724                                         msginfo->references =
725                                                 g_slist_prepend(msginfo->references, ref);
726                         }
727                         if (msginfo->references)
728                                 msginfo->references =
729                                         g_slist_reverse(msginfo->references);
730
731                         msginfo->folder = item;
732                         msginfo->flags.tmp_flags |= tmp_flags;
733
734                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
735                         if(msginfo->msgid)
736                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
737                 }
738         }
739 bail_err:
740         fclose(fp);
741
742         if (conv != NULL) {
743                 if (conv->free != NULL)
744                         conv->free(conv);
745                 g_free(conv);
746         }
747
748         if(error) {
749                 msgcache_destroy(cache);
750                 return NULL;
751         }
752
753         cache->last_access = time(NULL);
754         cache->memusage = memusage;
755
756         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
757         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
758         return cache;
759 }
760
761 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
762 {
763         FILE *fp;
764         MsgInfo *msginfo;
765         MsgPermFlags perm_flags;
766         guint32 num;
767         gint map_len = -1;
768         char *cache_data = NULL;
769         struct stat st;
770         
771         swapping = TRUE;
772
773         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
774          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
775          * it means it's the old version (not little-endian) on a big-endian machine. The code has
776          * no effect on x86 as their file doesn't change. */
777
778         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
779                 /* see if it isn't swapped ? */
780                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
781                         return;
782                 else
783                         swapping = FALSE; /* yay */
784         }
785         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
786         
787         if (msgcache_use_mmap_read) {
788                 if (fstat(fileno(fp), &st) >= 0)
789                         map_len = st.st_size;
790                 else
791                         map_len = -1;
792                 if (map_len > 0)
793                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
794         } else {
795                 cache_data = NULL;
796         }
797         if (cache_data != NULL && cache_data != MAP_FAILED) {
798                 int rem_len = map_len-ftell(fp);
799                 char *walk_data = cache_data+ftell(fp);
800
801                 while(rem_len > 0) {
802                         GET_CACHE_DATA_INT(num);
803                         GET_CACHE_DATA_INT(perm_flags);
804                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
805                         if(msginfo) {
806                                 msginfo->flags.perm_flags = perm_flags;
807                         }
808                 }
809                 munmap(cache_data, map_len);
810         } else {
811                 while (fread(&num, sizeof(num), 1, fp) == 1) {
812                         if (swapping)
813                                 num = bswap_32(num);
814                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) break;
815                         if (swapping)
816                                 perm_flags = bswap_32(perm_flags);
817                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
818                         if(msginfo) {
819                                 msginfo->flags.perm_flags = perm_flags;
820                         }
821                 }       
822         }
823         fclose(fp);
824 }
825
826 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
827 {
828         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
829         GSList *cur;
830         int w_err = 0, wrote = 0;
831
832         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
833         WRITE_CACHE_DATA_INT(msginfo->size, fp);
834         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
835         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
836         WRITE_CACHE_DATA_INT(flags, fp);
837
838         WRITE_CACHE_DATA(msginfo->fromname, fp);
839
840         WRITE_CACHE_DATA(msginfo->date, fp);
841         WRITE_CACHE_DATA(msginfo->from, fp);
842         WRITE_CACHE_DATA(msginfo->to, fp);
843         WRITE_CACHE_DATA(msginfo->cc, fp);
844         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
845         WRITE_CACHE_DATA(msginfo->subject, fp);
846         WRITE_CACHE_DATA(msginfo->msgid, fp);
847         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
848         WRITE_CACHE_DATA(msginfo->xref, fp);
849         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
850         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
851         
852         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
853
854         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
855                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
856         }
857         return w_err ? -1 : wrote;
858 }
859
860 static int msgcache_write_mmap_cache(MsgInfo *msginfo, char *walk_data)
861 {
862         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
863         GSList *cur;
864         int wrote = 0;
865
866         PUT_CACHE_DATA_INT(msginfo->msgnum);
867         PUT_CACHE_DATA_INT(msginfo->size);
868         PUT_CACHE_DATA_INT(msginfo->mtime);
869         PUT_CACHE_DATA_INT(msginfo->date_t);
870         PUT_CACHE_DATA_INT(flags);
871         PUT_CACHE_DATA(msginfo->fromname);
872
873         PUT_CACHE_DATA(msginfo->date);
874         PUT_CACHE_DATA(msginfo->from);
875         PUT_CACHE_DATA(msginfo->to);
876         PUT_CACHE_DATA(msginfo->cc);
877         PUT_CACHE_DATA(msginfo->newsgroups);
878         PUT_CACHE_DATA(msginfo->subject);
879         PUT_CACHE_DATA(msginfo->msgid);
880         PUT_CACHE_DATA(msginfo->inreplyto);
881         PUT_CACHE_DATA(msginfo->xref);
882         PUT_CACHE_DATA_INT(msginfo->planned_download);
883         PUT_CACHE_DATA_INT(msginfo->total_size);
884         
885         PUT_CACHE_DATA_INT(g_slist_length(msginfo->references));
886
887         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
888                 PUT_CACHE_DATA((gchar *)cur->data);
889         }
890         return wrote;
891 }
892
893 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
894 {
895         MsgPermFlags flags = msginfo->flags.perm_flags;
896         int w_err = 0, wrote = 0;
897         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
898         WRITE_CACHE_DATA_INT(flags, fp);
899         return w_err ? -1 : wrote;
900 }
901
902 static int msgcache_write_mmap_flags(MsgInfo *msginfo, char *walk_data)
903 {
904         MsgPermFlags flags = msginfo->flags.perm_flags;
905         int wrote = 0;
906
907         PUT_CACHE_DATA_INT(msginfo->msgnum);
908         PUT_CACHE_DATA_INT(flags);
909         return wrote;
910 }
911
912 struct write_fps
913 {
914         FILE *cache_fp;
915         FILE *mark_fp;
916         char *cache_data;
917         char *mark_data;
918         int error;
919         guint cache_size;
920         guint mark_size;
921 };
922
923 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
924 {
925         MsgInfo *msginfo;
926         struct write_fps *write_fps;
927         int tmp;
928
929         msginfo = (MsgInfo *)value;
930         write_fps = user_data;
931
932         tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
933         if (tmp < 0)
934                 write_fps->error = 1;
935         else
936                 write_fps->cache_size += tmp;
937         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
938         if (tmp < 0)
939                 write_fps->error = 1;
940         else
941                 write_fps->mark_size += tmp;
942 }
943
944 static void msgcache_write_mmap_func(gpointer key, gpointer value, gpointer user_data)
945 {
946         MsgInfo *msginfo;
947         struct write_fps *write_fps;
948         int tmp;
949
950         msginfo = (MsgInfo *)value;
951         write_fps = user_data;
952
953         tmp = msgcache_write_mmap_cache(msginfo, write_fps->cache_data);
954         write_fps->cache_size += tmp;
955         write_fps->cache_data += tmp;
956         tmp = msgcache_write_mmap_flags(msginfo, write_fps->mark_data);
957         write_fps->mark_size += tmp;
958         write_fps->mark_data += tmp;
959 }
960
961 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, MsgCache *cache)
962 {
963         struct write_fps write_fps;
964         gchar *new_cache, *new_mark;
965         int w_err = 0, wrote = 0;
966         gint map_len = -1;
967         char *cache_data = NULL;
968         char *mark_data = NULL;
969         START_TIMING("");
970         g_return_val_if_fail(cache_file != NULL, -1);
971         g_return_val_if_fail(mark_file != NULL, -1);
972         g_return_val_if_fail(cache != NULL, -1);
973
974         new_cache = g_strconcat(cache_file, ".new", NULL);
975         new_mark  = g_strconcat(mark_file, ".new", NULL);
976
977         write_fps.error = 0;
978         write_fps.cache_size = 0;
979         write_fps.mark_size = 0;
980         write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
981                 DATA_WRITE, NULL, 0);
982         if (write_fps.cache_fp == NULL) {
983                 g_free(new_cache);
984                 g_free(new_mark);
985                 return -1;
986         }
987
988         WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
989
990         if (w_err != 0) {
991                 g_warning("failed to write charset\n");
992                 fclose(write_fps.cache_fp);
993                 g_unlink(new_cache);
994                 g_free(new_cache);
995                 g_free(new_mark);
996                 return -1;
997         }
998
999         write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1000                 DATA_WRITE, NULL, 0);
1001         if (write_fps.mark_fp == NULL) {
1002                 fclose(write_fps.cache_fp);
1003                 g_unlink(new_cache);
1004                 g_free(new_cache);
1005                 g_free(new_mark);
1006                 return -1;
1007         }
1008
1009         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1010
1011         if (change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1012                 FILE_OP_ERROR(new_cache, "chmod");
1013
1014         write_fps.cache_size = ftell(write_fps.cache_fp);
1015         write_fps.mark_size = ftell(write_fps.mark_fp);
1016         if (msgcache_use_mmap_write && cache->memusage > 0) {
1017                 map_len = cache->memusage;
1018                 if (ftruncate(fileno(write_fps.cache_fp), (off_t)map_len) == 0) {
1019                         cache_data = mmap(NULL, map_len, PROT_WRITE, MAP_SHARED, 
1020                                 fileno(write_fps.cache_fp), 0);
1021                 }
1022                 if (cache_data != NULL && cache_data != MAP_FAILED) {
1023                         if (ftruncate(fileno(write_fps.mark_fp), (off_t)map_len) == 0) {
1024                                 mark_data = mmap(NULL, map_len, PROT_WRITE, MAP_SHARED, 
1025                                         fileno(write_fps.mark_fp), 0);
1026                         } 
1027                         if (mark_data == NULL || mark_data == MAP_FAILED) {
1028                                 munmap(cache_data, map_len);
1029                                 cache_data = NULL;
1030                         }
1031                 }
1032         }
1033
1034         if (cache_data != NULL && cache_data != MAP_FAILED) {
1035                 write_fps.cache_data = cache_data + ftell(write_fps.cache_fp);
1036                 write_fps.mark_data = mark_data + ftell(write_fps.mark_fp);
1037                 g_hash_table_foreach(cache->msgnum_table, msgcache_write_mmap_func, (gpointer)&write_fps);
1038                 munmap(cache_data, map_len);
1039                 munmap(mark_data, map_len);
1040                 ftruncate(fileno(write_fps.cache_fp), write_fps.cache_size);
1041                 ftruncate(fileno(write_fps.mark_fp), write_fps.mark_size);
1042         } else {
1043 #ifdef HAVE_FWRITE_UNLOCKED
1044                 flockfile(write_fps.cache_fp);
1045                 flockfile(write_fps.mark_fp);
1046 #endif
1047                 g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1048 #ifdef HAVE_FWRITE_UNLOCKED
1049                 funlockfile(write_fps.mark_fp);
1050                 funlockfile(write_fps.cache_fp);
1051 #endif
1052         }
1053         
1054         fflush(write_fps.cache_fp);
1055         fflush(write_fps.mark_fp);
1056
1057 #if 0
1058         fsync(fileno(write_fps.cache_fp));
1059         fsync(fileno(write_fps.mark_fp));
1060 #endif
1061
1062         fclose(write_fps.cache_fp);
1063         fclose(write_fps.mark_fp);
1064
1065
1066         if (write_fps.error != 0) {
1067                 g_unlink(new_cache);
1068                 g_unlink(new_mark);
1069                 g_free(new_cache);
1070                 g_free(new_mark);
1071                 return -1;
1072         } else {
1073                 move_file(new_cache, cache_file, TRUE);
1074                 move_file(new_mark, mark_file, TRUE);
1075                 cache->last_access = time(NULL);
1076         }
1077
1078         g_free(new_cache);
1079         g_free(new_mark);
1080         debug_print("done.\n");
1081         END_TIMING();
1082         return 0;
1083 }
1084