src/msgcache.c

   1 /*
   2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
   3  * Copyright (C) 1999-2007 Hiroyuki Yamamoto & The Claws Mail Team
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  */
  19
  20 #ifdef HAVE_CONFIG_H
  21 #  include "config.h"
  22 #endif
  23
  24 #include "defs.h"
  25
  26 #define _GNU_SOURCE
  27 #include <stdio.h>
  28
  29 #include <glib.h>
  30 #include <glib/gi18n.h>
  31 #include <sys/mman.h>
  32 #include <sys/types.h>
  33 #include <sys/stat.h>
  34
  35 #include <time.h>
  36
  37 #include "msgcache.h"
  38 #include "utils.h"
  39 #include "procmsg.h"
  40 #include "codeconv.h"
  41 #include "timing.h"
  42
  43 #ifdef HAVE_FWRITE_UNLOCKED
  44 #define SC_FWRITE fwrite_unlocked
  45 #else
  46 #define SC_FWRITE fwrite
  47 #endif
  48
  49 #if G_BYTE_ORDER == G_BIG_ENDIAN
  50 #define bswap_32(x) \
  51      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
  52       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
  53
  54 #define MMAP_TO_GUINT32(x)      \
  55         (((x[3]&0xff)) |        \
  56          ((x[2]&0xff) << 8) |   \
  57          ((x[1]&0xff) << 16) |  \
  58          ((x[0]&0xff) << 24))
  59
  60 #define MMAP_TO_GUINT32_SWAPPED(x)      \
  61         (((x[0]&0xff)) |                \
  62          ((x[1]&0xff) << 8) |           \
  63          ((x[2]&0xff) << 16) |          \
  64          ((x[3]&0xff) << 24))
  65
  66 static gboolean msgcache_use_mmap_read = TRUE;
  67 static gboolean msgcache_use_mmap_write = FALSE;
  68
  69 #else
  70 #define bswap_32(x) (x)
  71
  72 #define MMAP_TO_GUINT32(x)      \
  73         (((x[0]&0xff)) |        \
  74          ((x[1]&0xff) << 8) |   \
  75          ((x[2]&0xff) << 16) |  \
  76          ((x[3]&0xff) << 24))
  77
  78 #define MMAP_TO_GUINT32_SWAPPED(x)      \
  79         (((x[0]&0xff)) |                \
  80          ((x[1]&0xff) << 8) |           \
  81          ((x[2]&0xff) << 16) |          \
  82          ((x[3]&0xff) << 24))
  83
  84 static gboolean msgcache_use_mmap_read = TRUE;
  85 static gboolean msgcache_use_mmap_write = FALSE;
  86 #endif
  87
  88 static gboolean swapping = TRUE;
  89
  90 typedef enum
  91 {
  92         DATA_READ,
  93         DATA_WRITE,
  94         DATA_APPEND
  95 } DataOpenMode;
  96
  97 struct _MsgCache {
  98         GHashTable      *msgnum_table;
  99         GHashTable      *msgid_table;
 100         guint            memusage;
 101         time_t           last_access;
 102 };
 103
 104 typedef struct _StringConverter StringConverter;
 105 struct _StringConverter {
 106         gchar *(*convert) (StringConverter *converter, gchar *srcstr);
 107         void   (*free)    (StringConverter *converter);
 108 };
 109
 110 typedef struct _StrdupConverter StrdupConverter;
 111 struct _StrdupConverter {
 112         StringConverter converter;
 113 };
 114
 115 typedef struct _CharsetConverter CharsetConverter;
 116 struct _CharsetConverter {
 117         StringConverter converter;
 118
 119         gchar *srccharset;
 120         gchar *dstcharset;
 121 };
 122
 123 MsgCache *msgcache_new(void)
 124 {
 125         MsgCache *cache;
 126
 127         cache = g_new0(MsgCache, 1),
 128         cache->msgnum_table = g_hash_table_new(g_int_hash, g_int_equal);
 129         cache->msgid_table = g_hash_table_new(g_str_hash, g_str_equal);
 130         cache->last_access = time(NULL);
 131
 132         return cache;
 133 }
 134
 135 static gboolean msgcache_msginfo_free_func(gpointer num, gpointer msginfo, gpointer user_data)
 136 {
 137         procmsg_msginfo_free((MsgInfo *)msginfo);
 138         return TRUE;
 139 }
 140
 141 void msgcache_destroy(MsgCache *cache)
 142 {
 143         g_return_if_fail(cache != NULL);
 144
 145         g_hash_table_foreach_remove(cache->msgnum_table, msgcache_msginfo_free_func, NULL);
 146         g_hash_table_destroy(cache->msgid_table);
 147         g_hash_table_destroy(cache->msgnum_table);
 148         g_free(cache);
 149 }
 150
 151 void msgcache_add_msg(MsgCache *cache, MsgInfo *msginfo)
 152 {
 153         MsgInfo *newmsginfo;
 154
 155         g_return_if_fail(cache != NULL);
 156         g_return_if_fail(msginfo != NULL);
 157
 158         newmsginfo = procmsg_msginfo_new_ref(msginfo);
 159         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
 160         if(newmsginfo->msgid != NULL)
 161                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
 162         cache->memusage += procmsg_msginfo_memusage(msginfo);
 163         cache->last_access = time(NULL);
 164
 165         debug_print("Cache size: %d messages, %d bytes\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
 166 }
 167
 168 void msgcache_remove_msg(MsgCache *cache, guint msgnum)
 169 {
 170         MsgInfo *msginfo;
 171
 172         g_return_if_fail(cache != NULL);
 173         g_return_if_fail(msgnum > 0);
 174
 175         msginfo = (MsgInfo *) g_hash_table_lookup(cache->msgnum_table, &msgnum);
 176         if(!msginfo)
 177                 return;
 178
 179         cache->memusage -= procmsg_msginfo_memusage(msginfo);
 180         if(msginfo->msgid)
 181                 g_hash_table_remove(cache->msgid_table, msginfo->msgid);
 182         g_hash_table_remove(cache->msgnum_table, &msginfo->msgnum);
 183         procmsg_msginfo_free(msginfo);
 184         cache->last_access = time(NULL);
 185
 186         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
 187 }
 188
 189 void msgcache_update_msg(MsgCache *cache, MsgInfo *msginfo)
 190 {
 191         MsgInfo *oldmsginfo, *newmsginfo;
 192
 193         g_return_if_fail(cache != NULL);
 194         g_return_if_fail(msginfo != NULL);
 195
 196         oldmsginfo = g_hash_table_lookup(cache->msgnum_table, &msginfo->msgnum);
 197         if(oldmsginfo && oldmsginfo->msgid)
 198                 g_hash_table_remove(cache->msgid_table, oldmsginfo->msgid);
 199         if (oldmsginfo) {
 200                 g_hash_table_remove(cache->msgnum_table, &oldmsginfo->msgnum);
 201                 cache->memusage -= procmsg_msginfo_memusage(oldmsginfo);
 202                 procmsg_msginfo_free(oldmsginfo);
 203         }
 204
 205         newmsginfo = procmsg_msginfo_new_ref(msginfo);
 206         g_hash_table_insert(cache->msgnum_table, &newmsginfo->msgnum, newmsginfo);
 207         if(newmsginfo->msgid)
 208                 g_hash_table_insert(cache->msgid_table, newmsginfo->msgid, newmsginfo);
 209         cache->memusage += procmsg_msginfo_memusage(newmsginfo);
 210         cache->last_access = time(NULL);
 211
 212         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
 213
 214         return;
 215 }
 216
 217 MsgInfo *msgcache_get_msg(MsgCache *cache, guint num)
 218 {
 219         MsgInfo *msginfo;
 220
 221         g_return_val_if_fail(cache != NULL, NULL);
 222
 223         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
 224         if(!msginfo)
 225                 return NULL;
 226         cache->last_access = time(NULL);
 227
 228         return procmsg_msginfo_new_ref(msginfo);
 229 }
 230
 231 MsgInfo *msgcache_get_msg_by_id(MsgCache *cache, const gchar *msgid)
 232 {
 233         MsgInfo *msginfo;
 234
 235         g_return_val_if_fail(cache != NULL, NULL);
 236         g_return_val_if_fail(msgid != NULL, NULL);
 237
 238         msginfo = g_hash_table_lookup(cache->msgid_table, msgid);
 239         if(!msginfo)
 240                 return NULL;
 241         cache->last_access = time(NULL);
 242
 243         return procmsg_msginfo_new_ref(msginfo);
 244 }
 245
 246 static void msgcache_get_msg_list_func(gpointer key, gpointer value, gpointer user_data)
 247 {
 248         MsgInfoList **listptr = user_data;
 249         MsgInfo *msginfo = value;
 250
 251         *listptr = g_slist_prepend(*listptr, procmsg_msginfo_new_ref(msginfo));
 252 }
 253
 254 MsgInfoList *msgcache_get_msg_list(MsgCache *cache)
 255 {
 256         MsgInfoList *msg_list = NULL;
 257         START_TIMING("");
 258         g_return_val_if_fail(cache != NULL, NULL);
 259
 260         g_hash_table_foreach((GHashTable *)cache->msgnum_table, msgcache_get_msg_list_func, (gpointer)&msg_list);
 261         cache->last_access = time(NULL);
 262
 263         msg_list = g_slist_reverse(msg_list);
 264         END_TIMING();
 265         return msg_list;
 266 }
 267
 268 time_t msgcache_get_last_access_time(MsgCache *cache)
 269 {
 270         g_return_val_if_fail(cache != NULL, 0);
 271
 272         return cache->last_access;
 273 }
 274
 275 gint msgcache_get_memory_usage(MsgCache *cache)
 276 {
 277         g_return_val_if_fail(cache != NULL, 0);
 278
 279         return cache->memusage;
 280 }
 281
 282 /*
 283  *  Cache saving functions
 284  */
 285
 286 #define READ_CACHE_DATA(data, fp, total_len) \
 287 { \
 288         if ((tmp_len = msgcache_read_cache_data_str(fp, &data, conv)) < 0) { \
 289                 procmsg_msginfo_free(msginfo); \
 290                 error = TRUE; \
 291                 goto bail_err; \
 292         } \
 293         total_len += tmp_len; \
 294 }
 295
 296 #define READ_CACHE_DATA_INT(n, fp) \
 297 { \
 298         guint32 idata; \
 299         size_t ni; \
 300  \
 301         if ((ni = fread(&idata, sizeof(idata), 1, fp)) != 1) { \
 302                 g_warning("read_int: Cache data corrupted, read %d of %d at " \
 303                           "offset %ld\n", ni, sizeof(idata), ftell(fp)); \
 304                 procmsg_msginfo_free(msginfo); \
 305                 error = TRUE; \
 306                 goto bail_err; \
 307         } else \
 308                 n = swapping ? bswap_32(idata) : (idata);\
 309 }
 310
 311 #define GET_CACHE_DATA_INT(n) \
 312 { \
 313         n = (swapping ? (MMAP_TO_GUINT32_SWAPPED(walk_data)):(MMAP_TO_GUINT32(walk_data))); \
 314         walk_data += 4; rem_len -= 4;                   \
 315 }
 316
 317 #define GET_CACHE_DATA(data, total_len) \
 318 { \
 319         GET_CACHE_DATA_INT(tmp_len);    \
 320         if ((tmp_len = msgcache_get_cache_data_str(walk_data, &data, tmp_len, conv)) < 0) { \
 321                 printf("error at rem_len:%d\n", rem_len);\
 322                 procmsg_msginfo_free(msginfo); \
 323                 error = TRUE; \
 324                 goto bail_err; \
 325         } \
 326         total_len += tmp_len; \
 327         walk_data += tmp_len; rem_len -= tmp_len; \
 328 }
 329
 330
 331 #define WRITE_CACHE_DATA_INT(n, fp)                     \
 332 {                                                       \
 333         guint32 idata;                                  \
 334                                                         \
 335         idata = (guint32)bswap_32(n);                   \
 336         if (SC_FWRITE(&idata, sizeof(idata), 1, fp) != 1)       \
 337                 w_err = 1;                              \
 338         wrote += 4;                                     \
 339 }
 340
 341 #define PUT_CACHE_DATA_INT(n)                           \
 342 {                                                       \
 343         walk_data[0]=(((guint32)n)&0x000000ff);                 \
 344         walk_data[1]=(((guint32)n)&0x0000ff00)>>8;              \
 345         walk_data[2]=(((guint32)n)&0x00ff0000)>>16;             \
 346         walk_data[3]=(((guint32)n)&0xff000000)>>24;             \
 347         walk_data += 4;                                 \
 348         wrote += 4;                                     \
 349 }
 350
 351 #define WRITE_CACHE_DATA(data, fp) \
 352 { \
 353         size_t len;                                     \
 354         if (data == NULL)                               \
 355                 len = 0;                                \
 356         else                                            \
 357                 len = strlen(data);                     \
 358         WRITE_CACHE_DATA_INT(len, fp);                  \
 359         if (w_err == 0 && len > 0) {                    \
 360                 if (SC_FWRITE(data, 1, len, fp) != len) \
 361                         w_err = 1;                      \
 362                 wrote += len;                           \
 363         } \
 364 }
 365
 366 #define PUT_CACHE_DATA(data)                            \
 367 {                                                       \
 368         size_t len;                                     \
 369         if (data == NULL)                               \
 370                 len = 0;                                \
 371         else                                            \
 372                 len = strlen(data);                     \
 373         PUT_CACHE_DATA_INT(len);                        \
 374         if (len > 0) {                                  \
 375                 memcpy(walk_data, data, len);           \
 376                 walk_data += len;                       \
 377                 wrote += len;                           \
 378         }                                               \
 379 }
 380
 381 static FILE *msgcache_open_data_file(const gchar *file, guint version,
 382                                      DataOpenMode mode,
 383                                      gchar *buf, size_t buf_size)
 384 {
 385         FILE *fp;
 386         gint32 data_ver;
 387
 388         g_return_val_if_fail(file != NULL, NULL);
 389
 390         if (mode == DATA_WRITE) {
 391                 int w_err = 0, wrote = 0;
 392                 if ((fp = g_fopen(file, "w+")) == NULL) {
 393                         FILE_OP_ERROR(file, "fopen");
 394                         return NULL;
 395                 }
 396                 if (change_file_mode_rw(fp, file) < 0)
 397                         FILE_OP_ERROR(file, "chmod");
 398
 399                 WRITE_CACHE_DATA_INT(version, fp);
 400                 if (w_err != 0) {
 401                         g_warning("failed to write int\n");
 402                         fclose(fp);
 403                         return NULL;
 404                 }
 405                 return fp;
 406         }
 407
 408         /* check version */
 409         if ((fp = g_fopen(file, "rb")) == NULL)
 410                 debug_print("Mark/Cache file '%s' not found\n", file);
 411         else {
 412                 if (buf && buf_size > 0)
 413                         setvbuf(fp, buf, _IOFBF, buf_size);
 414                 if (fread(&data_ver, sizeof(data_ver), 1, fp) != 1 ||
 415                          version != bswap_32(data_ver)) {
 416                         g_message("%s: Mark/Cache version is different (%u != %u).\n",
 417                                   file, bswap_32(data_ver), version);
 418                         fclose(fp);
 419                         fp = NULL;
 420                 }
 421                 data_ver = bswap_32(data_ver);
 422         }
 423
 424         if (mode == DATA_READ)
 425                 return fp;
 426
 427         if (fp) {
 428                 /* reopen with append mode */
 429                 fclose(fp);
 430                 if ((fp = g_fopen(file, "ab")) == NULL)
 431                         FILE_OP_ERROR(file, "fopen");
 432         } else {
 433                 /* open with overwrite mode if mark file doesn't exist or
 434                    version is different */
 435                 fp = msgcache_open_data_file(file, version, DATA_WRITE, buf,
 436                                             buf_size);
 437         }
 438
 439         return fp;
 440 }
 441
 442 static gint msgcache_read_cache_data_str(FILE *fp, gchar **str,
 443                                          StringConverter *conv)
 444 {
 445         gchar *tmpstr = NULL;
 446         size_t ni;
 447         guint32 len;
 448
 449         *str = NULL;
 450         if (!swapping) {
 451                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
 452                     len > G_MAXINT) {
 453                         g_warning("read_data_str: Cache data (len) corrupted, read %d "
 454                                   "of %d bytes at offset %ld\n", ni, sizeof(len),
 455                                   ftell(fp));
 456                         return -1;
 457                 }
 458         } else {
 459                 if ((ni = fread(&len, sizeof(len), 1, fp) != 1) ||
 460                     bswap_32(len) > G_MAXINT) {
 461                         g_warning("read_data_str: Cache data (len) corrupted, read %d "
 462                                   "of %d bytes at offset %ld\n", ni, sizeof(len),
 463                                   ftell(fp));
 464                         return -1;
 465                 }
 466                 len = bswap_32(len);
 467         }
 468
 469         if (len == 0)
 470                 return 0;
 471
 472         tmpstr = g_try_malloc(len + 1);
 473
 474         if(tmpstr == NULL) {
 475                 g_warning("read_data_str: can't g_malloc %d bytes - cache data probably corrupted.\n", len);
 476                 return -1;
 477         }
 478
 479         if ((ni = fread(tmpstr, 1, len, fp)) != len) {
 480                 g_warning("read_data_str: Cache data corrupted, read %d of %d "
 481                           "bytes at offset %ld\n",
 482                           ni, len, ftell(fp));
 483                 g_free(tmpstr);
 484                 return -1;
 485         }
 486         tmpstr[len] = 0;
 487
 488         if (conv != NULL) {
 489                 *str = conv->convert(conv, tmpstr);
 490                 g_free(tmpstr);
 491         } else
 492                 *str = tmpstr;
 493
 494         return len;
 495 }
 496
 497 static gint msgcache_get_cache_data_str(gchar *src, gchar **str, gint len,
 498                                          StringConverter *conv)
 499 {
 500         gchar *tmpstr = NULL;
 501
 502         *str = NULL;
 503
 504         if (len == 0)
 505                 return 0;
 506
 507         if(len > 2*1024*1024) {
 508                 g_warning("read_data_str: refusing to allocate %d bytes.\n", len);
 509                 return -1;
 510         }
 511
 512         tmpstr = g_try_malloc(len + 1);
 513
 514         if(tmpstr == NULL) {
 515                 g_warning("read_data_str: can't g_malloc %d bytes - cache data probably corrupted.\n", len);
 516                 return -1;
 517         }
 518
 519         strncpy(tmpstr, src, len);
 520
 521         tmpstr[len] = 0;
 522
 523         if (conv != NULL) {
 524                 *str = conv->convert(conv, tmpstr);
 525                 g_free(tmpstr);
 526         } else
 527                 *str = tmpstr;
 528
 529         return len;
 530 }
 531
 532 static gchar *strconv_charset_convert(StringConverter *conv, gchar *srcstr)
 533 {
 534         CharsetConverter *charsetconv = (CharsetConverter *) conv;
 535
 536         return conv_codeset_strdup(srcstr, charsetconv->srccharset, charsetconv->dstcharset);
 537 }
 538
 539 static void strconv_charset_free(StringConverter *conv)
 540 {
 541         CharsetConverter *charsetconv = (CharsetConverter *) conv;
 542
 543         g_free(charsetconv->srccharset);
 544         g_free(charsetconv->dstcharset);
 545 }
 546
 547 MsgCache *msgcache_read_cache(FolderItem *item, const gchar *cache_file)
 548 {
 549         MsgCache *cache;
 550         FILE *fp;
 551         MsgInfo *msginfo;
 552         MsgTmpFlags tmp_flags = 0;
 553         gchar file_buf[BUFFSIZE];
 554         guint32 num;
 555         guint refnum;
 556         gboolean error = FALSE;
 557         StringConverter *conv = NULL;
 558         gchar *srccharset = NULL;
 559         const gchar *dstcharset = NULL;
 560         gchar *ref = NULL;
 561         guint memusage = 0;
 562         gint tmp_len = 0, map_len = -1;
 563         char *cache_data = NULL;
 564         struct stat st;
 565
 566         g_return_val_if_fail(cache_file != NULL, NULL);
 567         g_return_val_if_fail(item != NULL, NULL);
 568
 569         swapping = TRUE;
 570
 571         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
 572          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds,
 573          * it means it's the old version (not little-endian) on a big-endian machine. The code has
 574          * no effect on x86 as their file doesn't change. */
 575
 576         if ((fp = msgcache_open_data_file
 577                 (cache_file, CACHE_VERSION, DATA_READ, file_buf, sizeof(file_buf))) == NULL) {
 578                 if ((fp = msgcache_open_data_file
 579                 (cache_file, bswap_32(CACHE_VERSION), DATA_READ, file_buf, sizeof(file_buf))) == NULL)
 580                         return NULL;
 581                 else
 582                         swapping = FALSE;
 583         }
 584
 585         debug_print("\tReading %sswapped message cache from %s...\n", swapping?"":"un", cache_file);
 586
 587         if (folder_has_parent_of_type(item, F_QUEUE)) {
 588                 tmp_flags |= MSG_QUEUED;
 589         } else if (folder_has_parent_of_type(item, F_DRAFT)) {
 590                 tmp_flags |= MSG_DRAFT;
 591         }
 592
 593         if (msgcache_read_cache_data_str(fp, &srccharset, NULL) < 0)
 594                 return NULL;
 595         dstcharset = CS_UTF_8;
 596         if (srccharset == NULL || dstcharset == NULL) {
 597                 conv = NULL;
 598         } else if (strcmp(srccharset, dstcharset) == 0) {
 599                 debug_print("using Noop Converter\n");
 600
 601                 conv = NULL;
 602         } else {
 603                 CharsetConverter *charsetconv;
 604
 605                 debug_print("using CharsetConverter\n");
 606
 607                 charsetconv = g_new0(CharsetConverter, 1);
 608                 charsetconv->converter.convert = strconv_charset_convert;
 609                 charsetconv->converter.free = strconv_charset_free;
 610                 charsetconv->srccharset = g_strdup(srccharset);
 611                 charsetconv->dstcharset = g_strdup(dstcharset);
 612
 613                 conv = (StringConverter *) charsetconv;
 614         }
 615         g_free(srccharset);
 616
 617         cache = msgcache_new();
 618
 619         if (msgcache_use_mmap_read == TRUE) {
 620                 if (fstat(fileno(fp), &st) >= 0)
 621                         map_len = st.st_size;
 622                 else
 623                         map_len = -1;
 624                 if (map_len > 0)
 625                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
 626         } else {
 627                 cache_data = NULL;
 628         }
 629         if (cache_data != NULL && cache_data != MAP_FAILED) {
 630                 int rem_len = map_len-ftell(fp);
 631                 char *walk_data = cache_data+ftell(fp);
 632
 633                 while(rem_len > 0) {
 634                         GET_CACHE_DATA_INT(num);
 635
 636                         msginfo = procmsg_msginfo_new();
 637                         msginfo->msgnum = num;
 638                         memusage += sizeof(MsgInfo);
 639
 640                         GET_CACHE_DATA_INT(msginfo->size);
 641                         GET_CACHE_DATA_INT(msginfo->mtime);
 642                         GET_CACHE_DATA_INT(msginfo->date_t);
 643                         GET_CACHE_DATA_INT(msginfo->flags.tmp_flags);
 644
 645                         GET_CACHE_DATA(msginfo->fromname, memusage);
 646
 647                         GET_CACHE_DATA(msginfo->date, memusage);
 648                         GET_CACHE_DATA(msginfo->from, memusage);
 649                         GET_CACHE_DATA(msginfo->to, memusage);
 650                         GET_CACHE_DATA(msginfo->cc, memusage);
 651                         GET_CACHE_DATA(msginfo->newsgroups, memusage);
 652                         GET_CACHE_DATA(msginfo->subject, memusage);
 653                         GET_CACHE_DATA(msginfo->msgid, memusage);
 654                         GET_CACHE_DATA(msginfo->inreplyto, memusage);
 655                         GET_CACHE_DATA(msginfo->xref, memusage);
 656
 657                         GET_CACHE_DATA_INT(msginfo->planned_download);
 658                         GET_CACHE_DATA_INT(msginfo->total_size);
 659                         GET_CACHE_DATA_INT(refnum);
 660
 661                         for (; refnum != 0; refnum--) {
 662                                 ref = NULL;
 663
 664                                 GET_CACHE_DATA(ref, memusage);
 665
 666                                 if (ref && *ref)
 667                                         msginfo->references =
 668                                                 g_slist_prepend(msginfo->references, ref);
 669                         }
 670                         if (msginfo->references)
 671                                 msginfo->references =
 672                                         g_slist_reverse(msginfo->references);
 673
 674                         msginfo->folder = item;
 675                         msginfo->flags.tmp_flags |= tmp_flags;
 676
 677                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
 678                         if(msginfo->msgid)
 679                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
 680                 }
 681
 682                 munmap(cache_data, map_len);
 683         } else {
 684                 while (fread(&num, sizeof(num), 1, fp) == 1) {
 685                         if (swapping)
 686                                 num = bswap_32(num);
 687
 688                         msginfo = procmsg_msginfo_new();
 689                         msginfo->msgnum = num;
 690                         memusage += sizeof(MsgInfo);
 691
 692                         READ_CACHE_DATA_INT(msginfo->size, fp);
 693                         READ_CACHE_DATA_INT(msginfo->mtime, fp);
 694                         READ_CACHE_DATA_INT(msginfo->date_t, fp);
 695                         READ_CACHE_DATA_INT(msginfo->flags.tmp_flags, fp);
 696
 697                         READ_CACHE_DATA(msginfo->fromname, fp, memusage);
 698
 699                         READ_CACHE_DATA(msginfo->date, fp, memusage);
 700                         READ_CACHE_DATA(msginfo->from, fp, memusage);
 701                         READ_CACHE_DATA(msginfo->to, fp, memusage);
 702                         READ_CACHE_DATA(msginfo->cc, fp, memusage);
 703                         READ_CACHE_DATA(msginfo->newsgroups, fp, memusage);
 704                         READ_CACHE_DATA(msginfo->subject, fp, memusage);
 705                         READ_CACHE_DATA(msginfo->msgid, fp, memusage);
 706                         READ_CACHE_DATA(msginfo->inreplyto, fp, memusage);
 707                         READ_CACHE_DATA(msginfo->xref, fp, memusage);
 708
 709                         READ_CACHE_DATA_INT(msginfo->planned_download, fp);
 710                         READ_CACHE_DATA_INT(msginfo->total_size, fp);
 711                         READ_CACHE_DATA_INT(refnum, fp);
 712
 713                         for (; refnum != 0; refnum--) {
 714                                 ref = NULL;
 715
 716                                 READ_CACHE_DATA(ref, fp, memusage);
 717
 718                                 if (ref && *ref)
 719                                         msginfo->references =
 720                                                 g_slist_prepend(msginfo->references, ref);
 721                         }
 722                         if (msginfo->references)
 723                                 msginfo->references =
 724                                         g_slist_reverse(msginfo->references);
 725
 726                         msginfo->folder = item;
 727                         msginfo->flags.tmp_flags |= tmp_flags;
 728
 729                         g_hash_table_insert(cache->msgnum_table, &msginfo->msgnum, msginfo);
 730                         if(msginfo->msgid)
 731                                 g_hash_table_insert(cache->msgid_table, msginfo->msgid, msginfo);
 732                 }
 733         }
 734 bail_err:
 735         fclose(fp);
 736
 737         if (conv != NULL) {
 738                 if (conv->free != NULL)
 739                         conv->free(conv);
 740                 g_free(conv);
 741         }
 742
 743         if(error) {
 744                 msgcache_destroy(cache);
 745                 return NULL;
 746         }
 747
 748         cache->last_access = time(NULL);
 749         cache->memusage = memusage;
 750
 751         debug_print("done. (%d items read)\n", g_hash_table_size(cache->msgnum_table));
 752         debug_print("Cache size: %d messages, %d byte\n", g_hash_table_size(cache->msgnum_table), cache->memusage);
 753         return cache;
 754 }
 755
 756 void msgcache_read_mark(MsgCache *cache, const gchar *mark_file)
 757 {
 758         FILE *fp;
 759         MsgInfo *msginfo;
 760         MsgPermFlags perm_flags;
 761         guint32 num;
 762         gint map_len = -1;
 763         char *cache_data = NULL;
 764         struct stat st;
 765
 766         swapping = TRUE;
 767
 768         /* In case we can't open the mark file with MARK_VERSION, check if we can open it with the
 769          * swapped MARK_VERSION. As msgcache_open_data_file swaps it too, if this succeeds,
 770          * it means it's the old version (not little-endian) on a big-endian machine. The code has
 771          * no effect on x86 as their file doesn't change. */
 772
 773         if ((fp = msgcache_open_data_file(mark_file, MARK_VERSION, DATA_READ, NULL, 0)) == NULL) {
 774                 /* see if it isn't swapped ? */
 775                 if ((fp = msgcache_open_data_file(mark_file, bswap_32(MARK_VERSION), DATA_READ, NULL, 0)) == NULL)
 776                         return;
 777                 else
 778                         swapping = FALSE; /* yay */
 779         }
 780         debug_print("reading %sswapped mark file.\n", swapping?"":"un");
 781
 782         if (msgcache_use_mmap_read) {
 783                 if (fstat(fileno(fp), &st) >= 0)
 784                         map_len = st.st_size;
 785                 else
 786                         map_len = -1;
 787                 if (map_len > 0)
 788                         cache_data = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE, fileno(fp), 0);
 789         } else {
 790                 cache_data = NULL;
 791         }
 792         if (cache_data != NULL && cache_data != MAP_FAILED) {
 793                 int rem_len = map_len-ftell(fp);
 794                 char *walk_data = cache_data+ftell(fp);
 795
 796                 while(rem_len > 0) {
 797                         GET_CACHE_DATA_INT(num);
 798                         GET_CACHE_DATA_INT(perm_flags);
 799                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
 800                         if(msginfo) {
 801                                 msginfo->flags.perm_flags = perm_flags;
 802                         }
 803                 }
 804                 munmap(cache_data, map_len);
 805         } else {
 806                 while (fread(&num, sizeof(num), 1, fp) == 1) {
 807                         if (swapping)
 808                                 num = bswap_32(num);
 809                         if (fread(&perm_flags, sizeof(perm_flags), 1, fp) != 1) break;
 810                         if (swapping)
 811                                 perm_flags = bswap_32(perm_flags);
 812                         msginfo = g_hash_table_lookup(cache->msgnum_table, &num);
 813                         if(msginfo) {
 814                                 msginfo->flags.perm_flags = perm_flags;
 815                         }
 816                 }
 817         }
 818         fclose(fp);
 819 }
 820
 821 static int msgcache_write_cache(MsgInfo *msginfo, FILE *fp)
 822 {
 823         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
 824         GSList *cur;
 825         int w_err = 0, wrote = 0;
 826
 827         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
 828         WRITE_CACHE_DATA_INT(msginfo->size, fp);
 829         WRITE_CACHE_DATA_INT(msginfo->mtime, fp);
 830         WRITE_CACHE_DATA_INT(msginfo->date_t, fp);
 831         WRITE_CACHE_DATA_INT(flags, fp);
 832
 833         WRITE_CACHE_DATA(msginfo->fromname, fp);
 834
 835         WRITE_CACHE_DATA(msginfo->date, fp);
 836         WRITE_CACHE_DATA(msginfo->from, fp);
 837         WRITE_CACHE_DATA(msginfo->to, fp);
 838         WRITE_CACHE_DATA(msginfo->cc, fp);
 839         WRITE_CACHE_DATA(msginfo->newsgroups, fp);
 840         WRITE_CACHE_DATA(msginfo->subject, fp);
 841         WRITE_CACHE_DATA(msginfo->msgid, fp);
 842         WRITE_CACHE_DATA(msginfo->inreplyto, fp);
 843         WRITE_CACHE_DATA(msginfo->xref, fp);
 844         WRITE_CACHE_DATA_INT(msginfo->planned_download, fp);
 845         WRITE_CACHE_DATA_INT(msginfo->total_size, fp);
 846
 847         WRITE_CACHE_DATA_INT(g_slist_length(msginfo->references), fp);
 848
 849         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
 850                 WRITE_CACHE_DATA((gchar *)cur->data, fp);
 851         }
 852         return w_err ? -1 : wrote;
 853 }
 854
 855 static int msgcache_write_mmap_cache(MsgInfo *msginfo, char *walk_data)
 856 {
 857         MsgTmpFlags flags = msginfo->flags.tmp_flags & MSG_CACHED_FLAG_MASK;
 858         GSList *cur;
 859         int wrote = 0;
 860
 861         PUT_CACHE_DATA_INT(msginfo->msgnum);
 862         PUT_CACHE_DATA_INT(msginfo->size);
 863         PUT_CACHE_DATA_INT(msginfo->mtime);
 864         PUT_CACHE_DATA_INT(msginfo->date_t);
 865         PUT_CACHE_DATA_INT(flags);
 866         PUT_CACHE_DATA(msginfo->fromname);
 867
 868         PUT_CACHE_DATA(msginfo->date);
 869         PUT_CACHE_DATA(msginfo->from);
 870         PUT_CACHE_DATA(msginfo->to);
 871         PUT_CACHE_DATA(msginfo->cc);
 872         PUT_CACHE_DATA(msginfo->newsgroups);
 873         PUT_CACHE_DATA(msginfo->subject);
 874         PUT_CACHE_DATA(msginfo->msgid);
 875         PUT_CACHE_DATA(msginfo->inreplyto);
 876         PUT_CACHE_DATA(msginfo->xref);
 877         PUT_CACHE_DATA_INT(msginfo->planned_download);
 878         PUT_CACHE_DATA_INT(msginfo->total_size);
 879
 880         PUT_CACHE_DATA_INT(g_slist_length(msginfo->references));
 881
 882         for (cur = msginfo->references; cur != NULL; cur = cur->next) {
 883                 PUT_CACHE_DATA((gchar *)cur->data);
 884         }
 885         return wrote;
 886 }
 887
 888 static int msgcache_write_flags(MsgInfo *msginfo, FILE *fp)
 889 {
 890         MsgPermFlags flags = msginfo->flags.perm_flags;
 891         int w_err = 0, wrote = 0;
 892         WRITE_CACHE_DATA_INT(msginfo->msgnum, fp);
 893         WRITE_CACHE_DATA_INT(flags, fp);
 894         return w_err ? -1 : wrote;
 895 }
 896
 897 static int msgcache_write_mmap_flags(MsgInfo *msginfo, char *walk_data)
 898 {
 899         MsgPermFlags flags = msginfo->flags.perm_flags;
 900         int wrote = 0;
 901
 902         PUT_CACHE_DATA_INT(msginfo->msgnum);
 903         PUT_CACHE_DATA_INT(flags);
 904         return wrote;
 905 }
 906
 907 struct write_fps
 908 {
 909         FILE *cache_fp;
 910         FILE *mark_fp;
 911         char *cache_data;
 912         char *mark_data;
 913         int error;
 914         guint cache_size;
 915         guint mark_size;
 916 };
 917
 918 static void msgcache_write_func(gpointer key, gpointer value, gpointer user_data)
 919 {
 920         MsgInfo *msginfo;
 921         struct write_fps *write_fps;
 922         int tmp;
 923
 924         msginfo = (MsgInfo *)value;
 925         write_fps = user_data;
 926
 927         tmp = msgcache_write_cache(msginfo, write_fps->cache_fp);
 928         if (tmp < 0)
 929                 write_fps->error = 1;
 930         else
 931                 write_fps->cache_size += tmp;
 932         tmp= msgcache_write_flags(msginfo, write_fps->mark_fp);
 933         if (tmp < 0)
 934                 write_fps->error = 1;
 935         else
 936                 write_fps->mark_size += tmp;
 937 }
 938
 939 static void msgcache_write_mmap_func(gpointer key, gpointer value, gpointer user_data)
 940 {
 941         MsgInfo *msginfo;
 942         struct write_fps *write_fps;
 943         int tmp;
 944
 945         msginfo = (MsgInfo *)value;
 946         write_fps = user_data;
 947
 948         tmp = msgcache_write_mmap_cache(msginfo, write_fps->cache_data);
 949         write_fps->cache_size += tmp;
 950         write_fps->cache_data += tmp;
 951         tmp = msgcache_write_mmap_flags(msginfo, write_fps->mark_data);
 952         write_fps->mark_size += tmp;
 953         write_fps->mark_data += tmp;
 954 }
 955
 956 gint msgcache_write(const gchar *cache_file, const gchar *mark_file, MsgCache *cache)
 957 {
 958         struct write_fps write_fps;
 959         gchar *new_cache, *new_mark;
 960         int w_err = 0, wrote = 0;
 961         gint map_len = -1;
 962         char *cache_data = NULL;
 963         char *mark_data = NULL;
 964         START_TIMING("");
 965         g_return_val_if_fail(cache_file != NULL, -1);
 966         g_return_val_if_fail(mark_file != NULL, -1);
 967         g_return_val_if_fail(cache != NULL, -1);
 968
 969         new_cache = g_strconcat(cache_file, ".new", NULL);
 970         new_mark  = g_strconcat(mark_file, ".new", NULL);
 971
 972         write_fps.error = 0;
 973         write_fps.cache_size = 0;
 974         write_fps.mark_size = 0;
 975         write_fps.cache_fp = msgcache_open_data_file(new_cache, CACHE_VERSION,
 976                 DATA_WRITE, NULL, 0);
 977         if (write_fps.cache_fp == NULL) {
 978                 g_free(new_cache);
 979                 g_free(new_mark);
 980                 return -1;
 981         }
 982
 983         WRITE_CACHE_DATA(CS_UTF_8, write_fps.cache_fp);
 984
 985         if (w_err != 0) {
 986                 g_warning("failed to write charset\n");
 987                 fclose(write_fps.cache_fp);
 988                 g_unlink(new_cache);
 989                 g_free(new_cache);
 990                 g_free(new_mark);
 991                 return -1;
 992         }
 993
 994         write_fps.mark_fp = msgcache_open_data_file(new_mark, MARK_VERSION,
 995                 DATA_WRITE, NULL, 0);
 996         if (write_fps.mark_fp == NULL) {
 997                 fclose(write_fps.cache_fp);
 998                 g_unlink(new_cache);
 999                 g_free(new_cache);
1000                 g_free(new_mark);
1001                 return -1;
1002         }
1003
1004         debug_print("\tWriting message cache to %s and %s...\n", new_cache, new_mark);
1005
1006         if (change_file_mode_rw(write_fps.cache_fp, new_cache) < 0)
1007                 FILE_OP_ERROR(new_cache, "chmod");
1008
1009         write_fps.cache_size = ftell(write_fps.cache_fp);
1010         write_fps.mark_size = ftell(write_fps.mark_fp);
1011         if (msgcache_use_mmap_write && cache->memusage > 0) {
1012                 map_len = cache->memusage;
1013                 if (ftruncate(fileno(write_fps.cache_fp), (off_t)map_len) == 0) {
1014                         cache_data = mmap(NULL, map_len, PROT_WRITE, MAP_SHARED,
1015                                 fileno(write_fps.cache_fp), 0);
1016                 }
1017                 if (cache_data != NULL && cache_data != MAP_FAILED) {
1018                         if (ftruncate(fileno(write_fps.mark_fp), (off_t)map_len) == 0) {
1019                                 mark_data = mmap(NULL, map_len, PROT_WRITE, MAP_SHARED,
1020                                         fileno(write_fps.mark_fp), 0);
1021                         }
1022                         if (mark_data == NULL || mark_data == MAP_FAILED) {
1023                                 munmap(cache_data, map_len);
1024                                 cache_data = NULL;
1025                         }
1026                 }
1027         }
1028
1029         if (cache_data != NULL && cache_data != MAP_FAILED) {
1030                 write_fps.cache_data = cache_data + ftell(write_fps.cache_fp);
1031                 write_fps.mark_data = mark_data + ftell(write_fps.mark_fp);
1032                 g_hash_table_foreach(cache->msgnum_table, msgcache_write_mmap_func, (gpointer)&write_fps);
1033                 munmap(cache_data, map_len);
1034                 munmap(mark_data, map_len);
1035                 ftruncate(fileno(write_fps.cache_fp), write_fps.cache_size);
1036                 ftruncate(fileno(write_fps.mark_fp), write_fps.mark_size);
1037         } else {
1038 #ifdef HAVE_FWRITE_UNLOCKED
1039                 flockfile(write_fps.cache_fp);
1040                 flockfile(write_fps.mark_fp);
1041 #endif
1042                 g_hash_table_foreach(cache->msgnum_table, msgcache_write_func, (gpointer)&write_fps);
1043 #ifdef HAVE_FWRITE_UNLOCKED
1044                 funlockfile(write_fps.mark_fp);
1045                 funlockfile(write_fps.cache_fp);
1046 #endif
1047         }
1048
1049         fflush(write_fps.cache_fp);
1050         fflush(write_fps.mark_fp);
1051
1052 #if 0
1053         fsync(fileno(write_fps.cache_fp));
1054         fsync(fileno(write_fps.mark_fp));
1055 #endif
1056
1057         fclose(write_fps.cache_fp);
1058         fclose(write_fps.mark_fp);
1059
1060
1061         if (write_fps.error != 0) {
1062                 g_unlink(new_cache);
1063                 g_unlink(new_mark);
1064                 g_free(new_cache);
1065                 g_free(new_mark);
1066                 return -1;
1067         } else {
1068                 move_file(new_cache, cache_file, TRUE);
1069                 move_file(new_mark, mark_file, TRUE);
1070                 cache->last_access = time(NULL);
1071         }
1072
1073         g_free(new_cache);
1074         g_free(new_mark);
1075         debug_print("done.\n");
1076         END_TIMING();
1077         return 0;
1078 }
1079