0ee9da9687e92f65e55c72aa994ada3fa4c89461
[claws.git] / src / msgcache.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2007 Hiroyuki Yamamoto & The Claws Mail Team
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #endif
23
24 #include "defs.h"
25
26 #define _GNU_SOURCE
27 #include <stdio.h>
28
29 #include <glib.h>
30 #include <glib/gi18n.h>
31 #include <sys/mman.h>
32 #include <sys/types.h>
33 #include <sys/stat.h>
34
35 #include <time.h>
36
37 #include "msgcache.h"
38 #include "utils.h"
39 #include "procmsg.h"
40 #include "codeconv.h"
41 #include "timing.h"
42 #include "tags.h"
43
44 #ifdef HAVE_FWRITE_UNLOCKED
45 #define SC_FWRITE fwrite_unlocked
46 #else
47 #define SC_FWRITE fwrite
48 #endif
49
50 #if G_BYTE_ORDER == G_BIG_ENDIAN
51 #define bswap_32(x) \
52      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
53       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
54      
55 #define MMAP_TO_GUINT32(x)      \
56         (((x[3]&0xff)) |        \
57          ((x[2]&0xff) << 8) |   \
58          ((x[1]&0xff) << 16) |  \
59          ((x[0]&0xff) << 24))
60
61 #define MMAP_TO_GUINT32_SWAPPED(x)      \
62         (((x[0]&0xff)) |                \
63          ((x[1]&0xff) << 8) |           \
64          ((x[2]&0xff) << 16) |          \
65          ((x[3]&0xff) << 24))
66
67 static gboolean msgcache_use_mmap_read = TRUE;
68 static gboolean msgcache_use_mmap_write = FALSE;
69
70 #else
71 #define bswap_32(x) (x)
72
73 #define MMAP_TO_GUINT32(x)      \
74         (((x[0]&0xff)) |        \
75          ((x[1]&0xff) << 8) |   \
76          ((x[2]&0xff) << 16) |  \
77          ((x[3]&0xff) << 24))
78
79 #define MMAP_TO_GUINT32_SWAPPED(x)      \
80         (((x[0]&0xff)) |                \
81          ((x[1]&0xff) << 8) |           \
82          ((x[2]&0xff) << 16) |          \
83          ((x[3]&0xff) << 24))
84
85 static gboolean msgcache_use_mmap_read = TRUE;
86 static gboolean msgcache_use_mmap_write = FALSE;
87 #endif
88
89 static gboolean swapping = TRUE;
90
91 typedef enum
92 {
93         DATA_READ,
94         DATA_WRITE,
95         DATA_APPEND
96 } DataOpenMode;
97
98 struct _MsgCache {
99         GHashTable      *msgnum_table;
100         GHashTable      *msgid_table;
101         guint            memusage;
102         time_t           last_access;
103 };
104
105 typedef struct _StringConverter StringConverter;
106 struct _StringConverter {
107         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
108         void   (*free)    (StringConverter *converter);
109 };
110
111 typedef struct _StrdupConverter StrdupConverter;
112 struct _StrdupConverter {
113         StringConverter converter;
114 };
115
116 typedef struct _CharsetConverter CharsetConverter;
117 struct _CharsetConverter {
118         StringConverter converter;
119
120         gchar *srccharset;
121         gchar *dstcharset;
122 };
123
124 MsgCache *msgcache_new(void)
125 {
126         MsgCache *cache;
127         
128         cache = g_new0(MsgCache, 1),
129         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
130         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
131         cache->last_access = time(NULL);
132
133         return cache;
134 }
135
136 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
137 {
138         procmsg_msginfo_free((MsgInfo *)msginfo);
139         return TRUE;
140 }                                                                                         
141
142 void msgcache_destroy(MsgCache *cache)
143 {
144         g_return_if_fail(cache != NULL);
145
146         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
147         g_hash_table_destroy(cache->msgid_table);
148         g_hash_table_destroy(cache->msgnum_table);
149         g_free(cache);
150 }
151
152 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo) 
153 {
154         MsgInfo *newmsginfo;
155
156         g_return_if_fail(cache != NULL);
157         g_return_if_fail(msginfo != NULL);
158
159         newmsginfo = procmsg_msginfo_new_ref(msginfo);
160         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
161         if(newmsginfo->msgid != NULL)
162                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
163         cache->memusage += procmsg_msginfo_memusage(msginfo);
164         cache->last_access = time(NULL);
165
166         debug_print("Cache size: %d messages, %d bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
167 }
168
169 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
170 {
171         MsgInfo *msginfo;
172
173         g_return_if_fail(cache != NULL);
174         g_return_if_fail(msgnum > 0);
175
176         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
177         if(!msginfo)
178                 return;
179
180         cache->memusage -= procmsg_msginfo_memusage(msginfo);
181         if(msginfo->msgid)
182                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
183         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
184         procmsg_msginfo_free(msginfo);
185         cache->last_access = time(NULL);
186
187         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
188 }
189
190 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
191 {
192         MsgInfo *oldmsginfo, *newmsginfo;
193         
194         g_return_if_fail(cache != NULL);
195         g_return_if_fail(msginfo != NULL);
196
197         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
198         if(oldmsginfo && oldmsginfo->msgid) 
199                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
200         if (oldmsginfo) {
201                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
202                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
203                 procmsg_msginfo_free(oldmsginfo);
204         }
205
206         newmsginfo = procmsg_msginfo_new_ref(msginfo);
207         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
208         if(newmsginfo->msgid)
209                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
210         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
211         cache->last_access = time(NULL);
212         
213         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
214
215         return;
216 }
217
218 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
219 {
220         MsgInfo *msginfo;
221
222         g_return_val_if_fail(cache != NULL, NULL);
223
224         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
225         if(!msginfo)
226                 return NULL;
227         cache->last_access = time(NULL);
228         
229         return procmsg_msginfo_new_ref(msginfo);
230 }
231
232 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
233 {
234         MsgInfo *msginfo;
235         
236         g_return_val_if_fail(cache != NULL, NULL);
237         g_return_val_if_fail(msgid != NULL, NULL);
238
239         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
240         if(!msginfo)
241                 return NULL;
242         cache->last_access = time(NULL);
243         
244         return procmsg_msginfo_new_ref(msginfo);        
245 }
246
247 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
248 {
249         MsgInfoList **listptr = user_data;
250         MsgInfo *msginfo = value;
251
252         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
253 }
254
255 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
256 {
257         MsgInfoList *msg_list = NULL;
258         START_TIMING("");
259         g_return_val_if_fail(cache != NULL, NULL);
260
261         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);       
262         cache->last_access = time(NULL);
263         
264         msg_list = g_slist_reverse(msg_list);
265         END_TIMING();
266         return msg_list;
267 }
268
269 time_t msgcache_get_last_access_time(MsgCache *cache)
270 {
271         g_return_val_if_fail(cache != NULL, 0);
272         
273         return cache->last_access;
274 }
275
276 gint msgcache_get_memory_usage(MsgCache *cache)
277 {
278         g_return_val_if_fail(cache != NULL, 0);
279
280         return cache->memusage;
281 }
282
283 /*
284  *  Cache saving functions
285  */
286
287 #define READ_CACHE_DATA(data, fp, total_len) \
288 { \
289         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
290                 procmsg_msginfo_free(msginfo); \
291                 error = TRUE; \
292                 goto bail_err; \
293         } \
294         total_len += tmp_len; \
295 }
296
297 #define READ_CACHE_DATA_INT(n, fp) \
298 { \
299         guint32 idata; \
300         size_t ni; \
301  \
302         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
303                 g_warning("read_int: Cache data corrupted, read %d of %d at " \
304                           "offset %ld\n", ni, sizeof(idata), ftell(fp)); \
305                 procmsg_msginfo_free(msginfo); \
306                 error = TRUE; \
307                 goto bail_err; \
308         } else \
309                 n = swapping ? bswap_32(idata) : (idata);\
310 }
311
312 #define GET_CACHE_DATA_INT(n) \
313 { \
314         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data))); \
315         walk_data += 4; rem_len -= 4;                   \
316 }
317
318 #define GET_CACHE_DATA(data, total_len) \
319 { \
320         GET_CACHE_DATA_INT(tmp_len);    \
321         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
322                 printf("error at rem_len:%d\n", rem_len);\
323                 procmsg_msginfo_free(msginfo); \
324                 error = TRUE; \
325                 goto bail_err; \
326         } \
327         total_len += tmp_len; \
328         walk_data += tmp_len; rem_len -= tmp_len; \
329 }
330
331
332 #define WRITE_CACHE_DATA_INT(n, fp)                     \
333 {                                                       \
334         guint32 idata;                                  \
335                                                         \
336         idata = (guint32)bswap_32(n);                   \
337         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
338                 w_err = 1;                              \
339         wrote += 4;                                     \
340 }
341
342 #define PUT_CACHE_DATA_INT(n)                           \
343 {                                                       \
344         walk_data[0]=(((guint32)n)&0x000000ff);                 \
345         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
346         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
347         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
348         walk_data += 4;                                 \
349         wrote += 4;                                     \
350 }
351
352 #define WRITE_CACHE_DATA(data, fp) \
353 { \
354         size_t len;                                     \
355         if (data == NULL)                               \
356                 len = 0;                                \
357         else                                            \
358                 len = strlen(data);                     \
359         WRITE_CACHE_DATA_INT(len, fp);                  \
360         if (w_err == 0 && len > 0) {                    \
361                 if (SC_FWRITE(data, 1, len, fp) != len) \
362                         w_err = 1;                      \
363                 wrote += len;                           \
364         } \
365 }
366
367 #define PUT_CACHE_DATA(data)                            \
368 {                                                       \
369         size_t len;                                     \
370         if (data == NULL)                               \
371                 len = 0;                                \
372         else                                            \
373                 len = strlen(data);                     \
374         PUT_CACHE_DATA_INT(len);                        \
375         if (len > 0) {                                  \
376                 memcpy(walk_data, data, len);           \
377                 walk_data += len;                       \
378                 wrote += len;                           \
379         }                                               \
380 }
381
382 static FILE *msgcache_open_data_file(const gchar *file, guint version,
383                                      DataOpenMode mode,
384                                      gchar *buf, size_t buf_size)
385 {
386         FILE *fp;
387         gint32 data_ver;
388
389         g_return_val_if_fail(file != NULL, NULL);
390
391         if (mode == DATA_WRITE) {
392                 int w_err = 0, wrote = 0;
393                 if ((fp = g_fopen(file, "w+")) == NULL) {
394                         FILE_OP_ERROR(file, "fopen");
395                         return NULL;
396                 }
397                 if (change_file_mode_rw(fp, file) < 0)
398                         FILE_OP_ERROR(file, "chmod");
399
400                 WRITE_CACHE_DATA_INT(version, fp);
401                 if (w_err != 0) {
402                         g_warning("failed to write int\n");
403                         fclose(fp);
404                         return NULL;
405                 }
406                 return fp;
407         }
408
409         /* check version */
410         if ((fp = g_fopen(file, "rb")) == NULL)
411                 debug_print("Mark/Cache file '%s' not found\n", file);
412         else {
413                 if (buf && buf_size > 0)
414                         setvbuf(fp, buf, _IOFBF, buf_size);
415                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
416                          version != bswap_32(data_ver)) {
417                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
418                                   file, bswap_32(data_ver), version);
419                         fclose(fp);
420                         fp = NULL;
421                 }
422                 data_ver = bswap_32(data_ver);
423         }
424         
425         if (mode == DATA_READ)
426                 return fp;
427
428         if (fp) {
429                 /* reopen with append mode */
430                 fclose(fp);
431                 if ((fp = g_fopen(file, "ab")) == NULL)
432                         FILE_OP_ERROR(file, "fopen");
433         } else {
434                 /* open with overwrite mode if mark file doesn't exist or
435                    version is different */
436                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
437                                             buf_size);
438         }
439
440         return fp;
441 }
442
443 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str, 
444                                          StringConverter *conv)
445 {
446         gchar *tmpstr = NULL;
447         size_t ni;
448         guint32 len;
449
450         *str = NULL;
451         if (!swapping) {
452                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
453                     len > G_MAXINT) {
454                         g_warning("read_data_str: Cache data (len) corrupted, read %d "
455                                   "of %d bytes at offset %ld\n", ni, sizeof(len), 
456                                   ftell(fp));
457                         return -1;
458                 }
459         } else {
460                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
461                     bswap_32(len) > G_MAXINT) {
462                         g_warning("read_data_str: Cache data (len) corrupted, read %d "
463                                   "of %d bytes at offset %ld\n", ni, sizeof(len), 
464                                   ftell(fp));
465                         return -1;
466                 }
467                 len = bswap_32(len);
468         }
469
470         if (len == 0)
471                 return 0;
472
473         tmpstr = g_try_malloc(len + 1);
474
475         if(tmpstr == NULL) {
476                 g_warning("read_data_str: can't g_malloc %d bytes - cache data probably corrupted.\n", len);
477                 return -1;
478         }
479
480         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
481                 g_warning("read_data_str: Cache data corrupted, read %d of %d "
482                           "bytes at offset %ld\n", 
483                           ni, len, ftell(fp));
484                 g_free(tmpstr);
485                 return -1;
486         }
487         tmpstr[len] = 0;
488
489         if (conv != NULL) {
490                 *str = conv->convert(conv, tmpstr);
491                 g_free(tmpstr);
492         } else 
493                 *str = tmpstr;
494
495         return len;
496 }
497
498 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
499                                          StringConverter *conv)
500 {
501         gchar *tmpstr = NULL;
502
503         *str = NULL;
504
505         if (len == 0)
506                 return 0;
507
508         if(len > 2*1024*1024) {
509                 g_warning("read_data_str: refusing to allocate %d bytes.\n", len);
510                 return -1;
511         }
512
513         tmpstr = g_try_malloc(len + 1);
514
515         if(tmpstr == NULL) {
516                 g_warning("read_data_str: can't g_malloc %d bytes - cache data probably corrupted.\n", len);
517                 return -1;
518         }
519
520         strncpy(tmpstr, src, len);
521
522         tmpstr[len] = 0;
523
524         if (conv != NULL) {
525                 *str = conv->convert(conv, tmpstr);
526                 g_free(tmpstr);
527         } else 
528                 *str = tmpstr;
529
530         return len;
531 }
532
533 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
534 {
535         CharsetConverter *charsetconv = (CharsetConverter *) conv;
536
537         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
538 }
539
540 static void strconv_charset_free(StringConverter *conv)
541 {
542         CharsetConverter *charsetconv = (CharsetConverter *) conv;
543
544         g_free(charsetconv->srccharset);
545         g_free(charsetconv->dstcharset);
546 }
547
548 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
549 {
550         MsgCache *cache;
551         FILE *fp;
552         MsgInfo *msginfo;
553         MsgTmpFlags tmp_flags = 0;
554         gchar file_buf[BUFFSIZE];
555         guint32 num;
556         guint refnum;
557         gboolean error = FALSE;
558         StringConverter *conv = NULL;
559         gchar *srccharset = NULL;
560         const gchar *dstcharset = NULL;
561         gchar *ref = NULL;
562         guint memusage = 0;
563         gint tmp_len = 0, map_len = -1;
564         char *cache_data = NULL;
565         struct stat st;
566
567         g_return_val_if_fail(cache_file != NULL, NULL);
568         g_return_val_if_fail(item != NULL, NULL);
569
570         swapping = TRUE;
571
572         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
573          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
574          * it means it's the old version (not little-endian) on a big-endian machine. The code has
575          * no effect on x86 as their file doesn't change. */
576
577         if ((fp = msgcache_open_data_file
578                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
579                 if ((fp = msgcache_open_data_file
580                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
581                         return NULL;
582                 else
583                         swapping = FALSE;
584         }
585
586         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
587
588         if (folder_has_parent_of_type(item, F_QUEUE)) {
589                 tmp_flags |= MSG_QUEUED;
590         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
591                 tmp_flags |= MSG_DRAFT;
592         }
593
594         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0)
595                 return NULL;
596         dstcharset = CS_UTF_8;
597         if (srccharset == NULL || dstcharset == NULL) {
598                 conv = NULL;
599         } else if (strcmp(srccharset, dstcharset) == 0) {
600                 debug_print("using Noop Converter\n");
601
602                 conv = NULL;
603         } else {
604                 CharsetConverter *charsetconv;
605
606                 debug_print("using CharsetConverter\n");
607
608                 charsetconv = g_new0(CharsetConverter, 1);
609                 charsetconv->converter.convert = strconv_charset_convert;
610                 charsetconv->converter.free = strconv_charset_free;
611                 charsetconv->srccharset = g_strdup(srccharset);
612                 charsetconv->dstcharset = g_strdup(dstcharset);
613
614                 conv = (StringConverter *) charsetconv;
615         }
616         g_free(srccharset);
617
618         cache = msgcache_new();
619
620         if (msgcache_use_mmap_read == TRUE) {
621                 if (fstat(fileno(fp), &st) >= 0)
622                         map_len = st.st_size;
623                 else
624                         map_len = -1;
625                 if (map_len > 0)
626                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
627         } else {
628                 cache_data = NULL;
629         }
630         if (cache_data != NULL && cache_data != MAP_FAILED) {
631                 int rem_len = map_len-ftell(fp);
632                 char *walk_data = cache_data+ftell(fp);
633
634                 while(rem_len > 0) {
635                         GET_CACHE_DATA_INT(num);
636                         
637                         msginfo = procmsg_msginfo_new();
638                         msginfo->msgnum = num;
639                         memusage += sizeof(MsgInfo);
640
641                         GET_CACHE_DATA_INT(msginfo->size);
642                         GET_CACHE_DATA_INT(msginfo->mtime);
643                         GET_CACHE_DATA_INT(msginfo->date_t);
644                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
645
646                         GET_CACHE_DATA(msginfo->fromname, memusage);
647
648                         GET_CACHE_DATA(msginfo->date, memusage);
649                         GET_CACHE_DATA(msginfo->from, memusage);
650                         GET_CACHE_DATA(msginfo->to, memusage);
651                         GET_CACHE_DATA(msginfo->cc, memusage);
652                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
653                         GET_CACHE_DATA(msginfo->subject, memusage);
654                         GET_CACHE_DATA(msginfo->msgid, memusage);
655                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
656                         GET_CACHE_DATA(msginfo->xref, memusage);
657
658                         GET_CACHE_DATA_INT(msginfo->planned_download);
659                         GET_CACHE_DATA_INT(msginfo->total_size);
660                         GET_CACHE_DATA_INT(refnum);
661
662                         for (; refnum != 0; refnum--) {
663                                 ref = NULL;
664
665                                 GET_CACHE_DATA(ref, memusage);
666
667                                 if (ref && *ref)
668                                         msginfo->references =
669                                                 g_slist_prepend(msginfo->references, ref);
670                         }
671                         if (msginfo->references)
672                                 msginfo->references =
673                                         g_slist_reverse(msginfo->references);
674
675                         msginfo->folder = item;
676                         msginfo->flags.tmp_flags |= tmp_flags;
677
678                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
679                         if(msginfo->msgid)
680                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
681                 }
682                 
683                 munmap(cache_data, map_len);
684         } else {
685                 while (fread(&num, sizeof(num), 1, fp) == 1) {
686                         if (swapping)
687                                 num = bswap_32(num);
688
689                         msginfo = procmsg_msginfo_new();
690                         msginfo->msgnum = num;
691                         memusage += sizeof(MsgInfo);
692
693                         READ_CACHE_DATA_INT(msginfo->size, fp);
694                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
695                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
696                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
697
698                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
699
700                         READ_CACHE_DATA(msginfo->date, fp, memusage);
701                         READ_CACHE_DATA(msginfo->from, fp, memusage);
702                         READ_CACHE_DATA(msginfo->to, fp, memusage);
703                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
704                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
705                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
706                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
707                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
708                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
709
710                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
711                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
712                         READ_CACHE_DATA_INT(refnum, fp);
713
714                         for (; refnum != 0; refnum--) {
715                                 ref = NULL;
716
717                                 READ_CACHE_DATA(ref, fp, memusage);
718
719                                 if (ref && *ref)
720                                         msginfo->references =
721                                                 g_slist_prepend(msginfo->references, ref);
722                         }
723                         if (msginfo->references)
724                                 msginfo->references =
725                                         g_slist_reverse(msginfo->references);
726
727                         msginfo->folder = item;
728                         msginfo->flags.tmp_flags |= tmp_flags;
729
730                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
731                         if(msginfo->msgid)
732                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
733                 }
734         }
735 bail_err:
736         fclose(fp);
737
738         if (conv != NULL) {
739                 if (conv->free != NULL)
740                         conv->free(conv);
741                 g_free(conv);
742         }
743
744         if(error) {
745                 msgcache_destroy(cache);
746                 return NULL;
747         }
748
749         cache->last_access = time(NULL);
750         cache->memusage = memusage;
751
752         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
753         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
754         return cache;
755 }
756
757 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
758 {
759         FILE *fp;
760         MsgInfo *msginfo;
761         MsgPermFlags perm_flags;
762         guint32 num;
763         gint map_len = -1;
764         char *cache_data = NULL;
765         struct stat st;
766         
767         swapping = TRUE;
768
769         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
770          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
771          * it means it's the old version (not little-endian) on a big-endian machine. The code has
772          * no effect on x86 as their file doesn't change. */
773
774         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
775                 /* see if it isn't swapped ? */
776                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
777                         return;
778                 else
779                         swapping = FALSE; /* yay */
780         }
781         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
782         
783         if (msgcache_use_mmap_read) {
784                 if (fstat(fileno(fp), &st) >= 0)
785                         map_len = st.st_size;
786                 else
787                         map_len = -1;
788                 if (map_len > 0)
789                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
790         } else {
791                 cache_data = NULL;
792         }
793         if (cache_data != NULL && cache_data != MAP_FAILED) {
794                 int rem_len = map_len-ftell(fp);
795                 char *walk_data = cache_data+ftell(fp);
796
797                 while(rem_len > 0) {
798                         GET_CACHE_DATA_INT(num);
799                         GET_CACHE_DATA_INT(perm_flags);
800                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
801                         if(msginfo) {
802                                 msginfo->flags.perm_flags = perm_flags;
803                         }
804                 }
805                 munmap(cache_data, map_len);
806         } else {
807                 while (fread(&num, sizeof(num), 1, fp) == 1) {
808                         if (swapping)
809                                 num = bswap_32(num);
810                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) break;
811                         if (swapping)
812                                 perm_flags = bswap_32(perm_flags);
813                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
814                         if(msginfo) {
815                                 msginfo->flags.perm_flags = perm_flags;
816                         }
817                 }       
818         }
819         fclose(fp);
820 }
821
822 void msgcache_read_tags(MsgCache *cache, const gchar *tags_file)
823 {
824         FILE *fp;
825         MsgInfo *msginfo;
826         guint32 num;
827         gint map_len = -1;
828         char *cache_data = NULL;
829         struct stat st;
830         
831         swapping = TRUE;
832
833         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
834          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds, 
835          * it means it's the old version (not little-endian) on a big-endian machine. The code has
836          * no effect on x86 as their file doesn't change. */
837
838         if ((fp = msgcache_open_data_file(tags_file, TAGS_VERSION, DATA_READ, NULL, 0)) == NULL) {
839                 /* see if it isn't swapped ? */
840                 if ((fp = msgcache_open_data_file(tags_file, bswap_32(TAGS_VERSION), DATA_READ, NULL, 0)) == NULL)
841                         return;
842                 else
843                         swapping = FALSE; /* yay */
844         }
845         debug_print("reading %sswapped tags file.\n", swapping?"":"un");
846         
847         if (msgcache_use_mmap_read) {
848                 if (fstat(fileno(fp), &st) >= 0)
849                         map_len = st.st_size;
850                 else
851                         map_len = -1;
852                 if (map_len > 0)
853                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
854         } else {
855                 cache_data = NULL;
856         }
857         if (cache_data != NULL && cache_data != MAP_FAILED) {
858                 int rem_len = map_len-ftell(fp);
859                 char *walk_data = cache_data+ftell(fp);
860
861                 while(rem_len > 0) {
862                         gint id = -1;
863                         GET_CACHE_DATA_INT(num);
864                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
865                         if(msginfo) {
866                                 g_slist_free(msginfo->tags);
867                                 msginfo->tags = NULL;
868                                 do {
869                                         GET_CACHE_DATA_INT(id);
870                                         if (id > 0) {
871                                                 msginfo->tags = g_slist_prepend(
872                                                         msginfo->tags, 
873                                                         GINT_TO_POINTER(id));
874                                         }
875                                 } while (id > 0);
876                                 msginfo->tags = g_slist_reverse(msginfo->tags);
877                         }
878                 }
879                 munmap(cache_data, map_len);
880         } else {
881                 while (fread(&num, sizeof(num), 1, fp) == 1) {
882                         gint id = -1;
883                         if (swapping)
884                                 num = bswap_32(num);
885                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
886                         if(msginfo) {
887                                 g_slist_free(msginfo->tags);
888                                 msginfo->tags = NULL;
889                                 do {
890                                         if (fread(&id, sizeof(id), 1, fp) != 1) 
891                                                 id = -1;
892                                         if (swapping)
893                                                 id = bswap_32(id);
894                                         if (id > 0) {
895                                                 msginfo->tags = g_slist_prepend(
896                                                         msginfo->tags, 
897                                                         GINT_TO_POINTER(id));
898                                         }
899                                 } while (id > 0);
900                                 msginfo->tags = g_slist_reverse(msginfo->tags);
901                         }
902                 }
903         }
904         fclose(fp);
905 }
906
907 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
908 {
909         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
910         GSList *cur;
911         int w_err = 0, wrote = 0;
912
913         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
914         WRITE_CACHE_DATA_INT(msginfo->size, fp);
915         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
916         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
917         WRITE_CACHE_DATA_INT(flags, fp);
918
919         WRITE_CACHE_DATA(msginfo->fromname, fp);
920
921         WRITE_CACHE_DATA(msginfo->date, fp);
922         WRITE_CACHE_DATA(msginfo->from, fp);
923         WRITE_CACHE_DATA(msginfo->to, fp);
924         WRITE_CACHE_DATA(msginfo->cc, fp);
925         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
926         WRITE_CACHE_DATA(msginfo->subject, fp);
927         WRITE_CACHE_DATA(msginfo->msgid, fp);
928         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
929         WRITE_CACHE_DATA(msginfo->xref, fp);
930         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
931         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
932         
933         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
934
935         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
936                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
937         }
938         return w_err ? -1 : wrote;
939 }
940
941 static int msgcache_write_mmap_cache(MsgInfo *msginfo, char *walk_data)
942 {
943         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
944         GSList *cur;
945         int wrote = 0;
946
947         PUT_CACHE_DATA_INT(msginfo->msgnum);
948         PUT_CACHE_DATA_INT(msginfo->size);
949         PUT_CACHE_DATA_INT(msginfo->mtime);
950         PUT_CACHE_DATA_INT(msginfo->date_t);
951         PUT_CACHE_DATA_INT(flags);
952         PUT_CACHE_DATA(msginfo->fromname);
953
954         PUT_CACHE_DATA(msginfo->date);
955         PUT_CACHE_DATA(msginfo->from);
956         PUT_CACHE_DATA(msginfo->to);
957         PUT_CACHE_DATA(msginfo->cc);
958         PUT_CACHE_DATA(msginfo->newsgroups);
959         PUT_CACHE_DATA(msginfo->subject);
960         PUT_CACHE_DATA(msginfo->msgid);
961         PUT_CACHE_DATA(msginfo->inreplyto);
962         PUT_CACHE_DATA(msginfo->xref);
963         PUT_CACHE_DATA_INT(msginfo->planned_download);
964         PUT_CACHE_DATA_INT(msginfo->total_size);
965         
966         PUT_CACHE_DATA_INT(g_slist_length(msginfo->references));
967
968         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
969                 PUT_CACHE_DATA((gchar *)cur->data);
970         }
971         return wrote;
972 }
973
974 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
975 {
976         MsgPermFlags flags = msginfo->flags.perm_flags;
977         int w_err = 0, wrote = 0;
978         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
979         WRITE_CACHE_DATA_INT(flags, fp);
980         return w_err ? -1 : wrote;
981 }
982
983 static int msgcache_write_tags(MsgInfo *msginfo, FILE *fp)
984 {
985         GSList *cur = msginfo->tags;
986         int w_err = 0, wrote = 0;
987
988         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
989         for (; cur; cur = cur->next) {
990                 gint id = GPOINTER_TO_INT(cur->data);
991                 if (tags_get_tag(id) != NULL) {
992                         WRITE_CACHE_DATA_INT(id, fp);
993                 }
994         }
995         WRITE_CACHE_DATA_INT(-1, fp);
996
997         return w_err ? -1 : wrote;
998 }
999
1000 static int msgcache_write_mmap_flags(MsgInfo *msginfo, char *walk_data)
1001 {
1002         MsgPermFlags flags = msginfo->flags.perm_flags;
1003         int wrote = 0;
1004
1005         PUT_CACHE_DATA_INT(msginfo->msgnum);
1006         PUT_CACHE_DATA_INT(flags);
1007         return wrote;
1008 }
1009
1010 static int msgcache_write_mmap_tags(MsgInfo *msginfo, char *walk_data)
1011 {
1012         GSList *cur = msginfo->tags;
1013         int wrote = 0;
1014         
1015         PUT_CACHE_DATA_INT(msginfo->msgnum);
1016         for (; cur; cur = cur->next) {
1017                 gint id = GPOINTER_TO_INT(cur->data);
1018                 if (tags_get_tag(id) != NULL) {
1019                         PUT_CACHE_DATA_INT(id);
1020                 }
1021         }
1022         PUT_CACHE_DATA_INT(-1);
1023         return wrote;
1024 }
1025
1026 struct write_fps
1027 {
1028         FILE *cache_fp;
1029         FILE *mark_fp;
1030         FILE *tags_fp;
1031         char *cache_data;
1032         char *mark_data;
1033         char *tags_data;
1034         int error;
1035         guint cache_size;
1036         guint mark_size;
1037         guint tags_size;
1038 };
1039
1040 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
1041 {
1042         MsgInfo *msginfo;
1043         struct write_fps *write_fps;
1044         int tmp;
1045
1046         msginfo = (MsgInfo *)value;
1047         write_fps = user_data;
1048
1049         tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
1050         if (tmp < 0)
1051                 write_fps->error = 1;
1052         else
1053                 write_fps->cache_size += tmp;
1054         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
1055         if (tmp < 0)
1056                 write_fps->error = 1;
1057         else
1058                 write_fps->mark_size += tmp;
1059         tmp = msgcache_write_tags(msginfo, write_fps->tags_fp);
1060         if (tmp < 0)
1061                 write_fps->error = 1;
1062         else
1063                 write_fps->tags_size += tmp;
1064 }
1065
1066 static void msgcache_write_mmap_func(gpointer key, gpointer value, gpointer user_data)
1067 {
1068         MsgInfo *msginfo;
1069         struct write_fps *write_fps;
1070         int tmp;
1071
1072         msginfo = (MsgInfo *)value;
1073         write_fps = user_data;
1074
1075         tmp = msgcache_write_mmap_cache(msginfo, write_fps->cache_data);
1076         write_fps->cache_size += tmp;
1077         write_fps->cache_data += tmp;
1078         tmp = msgcache_write_mmap_flags(msginfo, write_fps->mark_data);
1079         write_fps->mark_size += tmp;
1080         write_fps->mark_data += tmp;
1081         tmp = msgcache_write_mmap_tags(msginfo, write_fps->tags_data);
1082         write_fps->tags_size += tmp;
1083         write_fps->tags_data += tmp;
1084 }
1085
1086 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, const gchar *tags_file, MsgCache *cache)
1087 {
1088         struct write_fps write_fps;
1089         gchar *new_cache, *new_mark, *new_tags;
1090         int w_err = 0, wrote = 0;
1091         gint map_len = -1;
1092         char *cache_data = NULL;
1093         char *mark_data = NULL;
1094         char *tags_data = NULL;
1095
1096         START_TIMING("");
1097         g_return_val_if_fail(cache_file != NULL, -1);
1098         g_return_val_if_fail(mark_file != NULL, -1);
1099         g_return_val_if_fail(tags_file != NULL, -1);
1100         g_return_val_if_fail(cache != NULL, -1);
1101
1102         new_cache = g_strconcat(cache_file, ".new", NULL);
1103         new_mark  = g_strconcat(mark_file, ".new", NULL);
1104         new_tags  = g_strconcat(tags_file, ".new", NULL);
1105
1106         write_fps.error = 0;
1107         write_fps.cache_size = 0;
1108         write_fps.mark_size = 0;
1109         write_fps.tags_size = 0;
1110
1111         write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
1112                 DATA_WRITE, NULL, 0);
1113         if (write_fps.cache_fp == NULL) {
1114                 g_free(new_cache);
1115                 g_free(new_mark);
1116                 return -1;
1117         }
1118
1119         WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
1120
1121         if (w_err != 0) {
1122                 g_warning("failed to write charset\n");
1123                 fclose(write_fps.cache_fp);
1124                 g_unlink(new_cache);
1125                 g_free(new_cache);
1126                 g_free(new_mark);
1127                 g_free(new_tags);
1128                 return -1;
1129         }
1130
1131         write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
1132                 DATA_WRITE, NULL, 0);
1133         if (write_fps.mark_fp == NULL) {
1134                 fclose(write_fps.cache_fp);
1135                 g_unlink(new_cache);
1136                 g_free(new_cache);
1137                 g_free(new_mark);
1138                 g_free(new_tags);
1139                 return -1;
1140         }
1141
1142         write_fps.tags_fp = msgcache_open_data_file(new_tags, TAGS_VERSION,
1143                 DATA_WRITE, NULL, 0);
1144         if (write_fps.tags_fp == NULL) {
1145                 fclose(write_fps.cache_fp);
1146                 fclose(write_fps.mark_fp);
1147                 g_unlink(new_cache);
1148                 g_unlink(new_mark);
1149                 g_free(new_cache);
1150                 g_free(new_mark);
1151                 g_free(new_tags);
1152                 return -1;
1153         }
1154
1155         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1156
1157         if (change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1158                 FILE_OP_ERROR(new_cache, "chmod");
1159
1160         write_fps.cache_size = ftell(write_fps.cache_fp);
1161         write_fps.mark_size = ftell(write_fps.mark_fp);
1162         write_fps.tags_size = ftell(write_fps.tags_fp);
1163
1164         if (msgcache_use_mmap_write && cache->memusage > 0) {
1165                 map_len = cache->memusage;
1166                 if (ftruncate(fileno(write_fps.cache_fp), (off_t)map_len) == 0) {
1167                         cache_data = mmap(NULL, map_len, PROT_WRITE, MAP_SHARED, 
1168                                 fileno(write_fps.cache_fp), 0);
1169                 }
1170                 if (cache_data != NULL && cache_data != MAP_FAILED) {
1171                         if (ftruncate(fileno(write_fps.mark_fp), (off_t)map_len) == 0) {
1172                                 mark_data = mmap(NULL, map_len, PROT_WRITE, MAP_SHARED, 
1173                                         fileno(write_fps.mark_fp), 0);
1174                         } 
1175                         if (mark_data == NULL || mark_data == MAP_FAILED) {
1176                                 munmap(cache_data, map_len);
1177                                 cache_data = NULL;
1178                         } else {
1179                                 if (ftruncate(fileno(write_fps.tags_fp), (off_t)map_len) == 0) {
1180                                         tags_data = mmap(NULL, map_len, PROT_WRITE, MAP_SHARED, 
1181                                                 fileno(write_fps.tags_fp), 0);
1182                                 } 
1183                                 if (tags_data == NULL || tags_data == MAP_FAILED) {
1184                                         munmap(cache_data, map_len);
1185                                         cache_data = NULL;
1186                                         munmap(mark_data, map_len);
1187                                         mark_data = NULL;
1188                                 } 
1189                         }
1190                 }
1191         }
1192
1193         if (cache_data != NULL && cache_data != MAP_FAILED) {
1194                 write_fps.cache_data = cache_data + ftell(write_fps.cache_fp);
1195                 write_fps.mark_data = mark_data + ftell(write_fps.mark_fp);
1196                 write_fps.tags_data = mark_data + ftell(write_fps.tags_fp);
1197                 g_hash_table_foreach(cache->msgnum_table, msgcache_write_mmap_func, (gpointer)&write_fps);
1198                 munmap(cache_data, map_len);
1199                 munmap(mark_data, map_len);
1200                 munmap(tags_data, map_len);
1201                 ftruncate(fileno(write_fps.cache_fp), write_fps.cache_size);
1202                 ftruncate(fileno(write_fps.mark_fp), write_fps.mark_size);
1203                 ftruncate(fileno(write_fps.tags_fp), write_fps.tags_size);
1204         } else {
1205 #ifdef HAVE_FWRITE_UNLOCKED
1206                 flockfile(write_fps.cache_fp);
1207                 flockfile(write_fps.mark_fp);
1208                 flockfile(write_fps.tags_fp);
1209 #endif
1210                 g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1211 #ifdef HAVE_FWRITE_UNLOCKED
1212                 funlockfile(write_fps.mark_fp);
1213                 funlockfile(write_fps.cache_fp);
1214                 funlockfile(write_fps.tags_fp);
1215 #endif
1216         }
1217         
1218         fflush(write_fps.cache_fp);
1219         fflush(write_fps.mark_fp);
1220         fflush(write_fps.tags_fp);
1221
1222 #if 0
1223         fsync(fileno(write_fps.cache_fp));
1224         fsync(fileno(write_fps.mark_fp));
1225         fsync(fileno(write_fps.tags_fp));
1226 #endif
1227
1228         fclose(write_fps.cache_fp);
1229         fclose(write_fps.mark_fp);
1230         fclose(write_fps.tags_fp);
1231
1232
1233         if (write_fps.error != 0) {
1234                 g_unlink(new_cache);
1235                 g_unlink(new_mark);
1236                 g_unlink(new_tags);
1237                 g_free(new_cache);
1238                 g_free(new_mark);
1239                 g_free(new_tags);
1240                 return -1;
1241         } else {
1242                 move_file(new_cache, cache_file, TRUE);
1243                 move_file(new_mark, mark_file, TRUE);
1244                 move_file(new_tags, tags_file, TRUE);
1245                 cache->last_access = time(NULL);
1246         }
1247
1248         g_free(new_cache);
1249         g_free(new_mark);
1250         g_free(new_tags);
1251         debug_print("done.\n");
1252         END_TIMING();
1253         return 0;
1254 }
1255