show progress while fetching news headers
[claws.git] / src / news.c
1 /*
2  * Sylpheed -- a GTK+ based, lightweight, and fast e-mail client
3  * Copyright (C) 1999-2002 Hiroyuki Yamamoto
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #  include "config.h"
22 #endif
23
24 #include "defs.h"
25
26 #include <glib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <stdlib.h>
30 #include <dirent.h>
31 #include <unistd.h>
32 #include <time.h>
33
34 #include "intl.h"
35 #include "news.h"
36 #include "nntp.h"
37 #include "socket.h"
38 #include "recv.h"
39 #include "procmsg.h"
40 #include "procheader.h"
41 #include "folder.h"
42 #include "session.h"
43 #include "statusbar.h"
44 #include "codeconv.h"
45 #include "utils.h"
46 #include "prefs_common.h"
47 #include "prefs_account.h"
48 #include "inputdialog.h"
49 #include "alertpanel.h"
50 #include "log.h"
51 #include "mainwindow.h"
52 #include "inc.h"
53 #if USE_OPENSSL
54 #  include "ssl.h"
55 #endif
56
57 #define NNTP_PORT       119
58 #if USE_OPENSSL
59 #define NNTPS_PORT      563
60 #endif
61
62 static void news_folder_init             (Folder        *folder,
63                                           const gchar   *name,
64                                           const gchar   *path);
65
66 #if USE_OPENSSL
67 static Session *news_session_new         (const gchar   *server,
68                                           gushort        port,
69                                           const gchar   *userid,
70                                           const gchar   *passwd,
71                                           SSLType        ssl_type);
72 #else
73 static Session *news_session_new         (const gchar   *server,
74                                           gushort        port,
75                                           const gchar   *userid,
76                                           const gchar   *passwd);
77 #endif
78
79 static gint news_get_article_cmd         (NNTPSession   *session,
80                                           const gchar   *cmd,
81                                           gint           num,
82                                           gchar         *filename);
83 static gint news_get_article             (NNTPSession   *session,
84                                           gint           num,
85                                           gchar         *filename);
86
87 static gint news_select_group            (NNTPSession   *session,
88                                           const gchar   *group,
89                                           gint          *num,
90                                           gint          *first,
91                                           gint          *last);
92 static GSList *news_get_uncached_articles(NNTPSession   *session,
93                                           FolderItem    *item,
94                                           gint           cache_last,
95                                           gint          *rfirst,
96                                           gint          *rlast);
97 static MsgInfo *news_parse_xover         (const gchar   *xover_str);
98 static gchar *news_parse_xhdr            (const gchar   *xhdr_str,
99                                           MsgInfo       *msginfo);
100 static GSList *news_delete_old_articles  (GSList        *alist,
101                                           FolderItem    *item,
102                                           gint           first);
103 static void news_delete_all_articles     (FolderItem    *item);
104 static void news_delete_expired_caches   (GSList        *alist,
105                                           FolderItem    *item);
106
107 static gint news_remove_msg              (Folder        *folder, 
108                                           FolderItem    *item, 
109                                           gint           num);
110 gint news_get_num_list                   (Folder        *folder, 
111                                           FolderItem    *item,
112                                           GSList       **list);
113 MsgInfo *news_get_msginfo                (Folder        *folder, 
114                                           FolderItem    *item,
115                                           gint           num);
116 GSList *news_get_msginfos                (Folder        *folder,
117                                           FolderItem    *item,
118                                           GSList        *msgnum_list);
119
120 gint news_post_stream                    (Folder        *folder, 
121                                           FILE          *fp);
122
123 Folder *news_folder_new(const gchar *name, const gchar *path)
124 {
125         Folder *folder;
126
127         folder = (Folder *)g_new0(NewsFolder, 1);
128         news_folder_init(folder, name, path);
129
130         return folder;
131 }
132
133 void news_folder_destroy(Folder *folder)
134 {
135         gchar *dir;
136
137         dir = folder_get_path(folder);
138         if (is_dir_exist(dir))
139                 remove_dir_recursive(dir);
140         g_free(dir);
141
142         folder_remote_folder_destroy(REMOTE_FOLDER(folder));
143 }
144
145 static void news_folder_init(Folder *folder, const gchar *name,
146                              const gchar *path)
147 {
148         folder->type = F_NEWS;
149
150         folder_remote_folder_init(folder, name, path);
151
152 /*
153         folder->get_msg_list = news_get_article_list;
154 */
155         folder->fetch_msg    = news_fetch_msg;
156 /*
157         folder->scan         = news_scan_group;
158 */
159         folder->destroy      = news_folder_destroy;
160         folder->remove_msg   = news_remove_msg;
161         folder->get_num_list = news_get_num_list;
162         folder->get_msginfo  = news_get_msginfo;
163         folder->get_msginfos = news_get_msginfos;
164 }
165
166 #if USE_OPENSSL
167 static Session *news_session_new(const gchar *server, gushort port,
168                                  const gchar *userid, const gchar *passwd,
169                                  SSLType ssl_type)
170 #else
171 static Session *news_session_new(const gchar *server, gushort port,
172                                  const gchar *userid, const gchar *passwd)
173 #endif
174 {
175         gchar buf[NNTPBUFSIZE];
176         NNTPSession *session;
177         NNTPSockInfo *nntp_sock;
178
179         g_return_val_if_fail(server != NULL, NULL);
180
181         log_message(_("creating NNTP connection to %s:%d ...\n"), server, port);
182
183 #if USE_OPENSSL
184         if (userid && passwd)
185                 nntp_sock = nntp_open_auth(server, port, buf, userid, passwd,
186                                            ssl_type);
187         else
188                 nntp_sock = nntp_open(server, port, buf, ssl_type);
189 #else
190         if (userid && passwd)
191                 nntp_sock = nntp_open_auth(server, port, buf, userid, passwd);
192         else
193                 nntp_sock = nntp_open(server, port, buf);
194 #endif
195
196         if (nntp_sock == NULL)
197                 return NULL;
198
199         session = g_new(NNTPSession, 1);
200         SESSION(session)->type             = SESSION_NEWS;
201         SESSION(session)->server           = g_strdup(server);
202         session->nntp_sock                 = nntp_sock;
203         SESSION(session)->sock             = nntp_sock->sock;
204         SESSION(session)->connected        = TRUE;
205         SESSION(session)->phase            = SESSION_READY;
206         SESSION(session)->last_access_time = time(NULL);
207         SESSION(session)->data             = NULL;
208
209         SESSION(session)->destroy          = news_session_destroy;
210
211         session->group = NULL;
212
213         return SESSION(session);
214 }
215
216 void news_session_destroy(Session *session)
217 {
218         nntp_close(NNTP_SESSION(session)->nntp_sock);
219         NNTP_SESSION(session)->nntp_sock = NULL;
220         session->sock = NULL;
221
222         g_free(NNTP_SESSION(session)->group);
223 }
224
225 static Session *news_session_new_for_folder(Folder *folder)
226 {
227         Session *session;
228         PrefsAccount *ac;
229         const gchar *userid = NULL;
230         gchar *passwd = NULL;
231         gushort port;
232         gchar buf[NNTPBUFSIZE];
233
234         g_return_val_if_fail(folder != NULL, NULL);
235         g_return_val_if_fail(folder->account != NULL, NULL);
236
237         ac = folder->account;
238         if (ac->use_nntp_auth && ac->userid && ac->userid[0]) {
239                 userid = ac->userid;
240                 if (ac->passwd && ac->passwd[0])
241                         passwd = g_strdup(ac->passwd);
242                 else
243                         passwd = input_dialog_query_password(ac->nntp_server,
244                                                              userid);
245         }
246
247 #if USE_OPENSSL
248         port = ac->set_nntpport ? ac->nntpport
249                 : ac->ssl_nntp ? NNTPS_PORT : NNTP_PORT;
250         session = news_session_new(ac->nntp_server, port, userid, passwd,
251                                    ac->ssl_nntp);
252 #else
253         port = ac->set_nntpport ? ac->nntpport : NNTP_PORT;
254         session = news_session_new(ac->nntp_server, port, userid, passwd);
255 #endif
256         if (ac->use_nntp_auth && ac->use_nntp_auth_onconnect)
257                 nntp_forceauth(NNTP_SESSION(session)->nntp_sock, buf, userid, passwd);
258
259         g_free(passwd);
260
261         return session;
262 }
263
264 NNTPSession *news_session_get(Folder *folder)
265 {
266         RemoteFolder *rfolder = REMOTE_FOLDER(folder);
267
268         g_return_val_if_fail(folder != NULL, NULL);
269         g_return_val_if_fail(folder->type == F_NEWS, NULL);
270         g_return_val_if_fail(folder->account != NULL, NULL);
271
272         if (!rfolder->session) {
273                 rfolder->session = news_session_new_for_folder(folder);
274                 return NNTP_SESSION(rfolder->session);
275         }
276
277         if (time(NULL) - rfolder->session->last_access_time < SESSION_TIMEOUT) {
278                 rfolder->session->last_access_time = time(NULL);
279                 return NNTP_SESSION(rfolder->session);
280         }
281
282         if (nntp_mode(NNTP_SESSION(rfolder->session)->nntp_sock, FALSE)
283             != NN_SUCCESS) {
284                 log_warning("NNTP connection to %s:%d has been"
285                               " disconnected. Reconnecting...\n",
286                             folder->account->nntp_server,
287                             folder->account->set_nntpport ?
288                             folder->account->nntpport : NNTP_PORT);
289                 session_destroy(rfolder->session);
290                 rfolder->session = news_session_new_for_folder(folder);
291         }
292
293         if (rfolder->session)
294                 rfolder->session->last_access_time = time(NULL);
295         return NNTP_SESSION(rfolder->session);
296 }
297
298 GSList *news_get_article_list(Folder *folder, FolderItem *item,
299                               gboolean use_cache)
300 {
301         GSList *alist;
302         NNTPSession *session;
303
304         g_return_val_if_fail(folder != NULL, NULL);
305         g_return_val_if_fail(item != NULL, NULL);
306         g_return_val_if_fail(folder->type == F_NEWS, NULL);
307
308         session = news_session_get(folder);
309
310         if (!session) {
311                 alist = procmsg_read_cache(item, FALSE);
312                 item->last_num = procmsg_get_last_num_in_msg_list(alist);
313         } else if (use_cache) {
314                 GSList *newlist;
315                 gint cache_last;
316                 gint first, last;
317
318                 alist = procmsg_read_cache(item, FALSE);
319
320                 cache_last = procmsg_get_last_num_in_msg_list(alist);
321                 newlist = news_get_uncached_articles
322                         (session, item, cache_last, &first, &last);
323                 if (first == 0 && last == 0) {
324                         news_delete_all_articles(item);
325                         procmsg_msg_list_free(alist);
326                         alist = NULL;
327                 } else {
328                         alist = news_delete_old_articles(alist, item, first);
329                         news_delete_expired_caches(alist, item);
330                 }
331
332                 alist = g_slist_concat(alist, newlist);
333
334                 item->last_num = last;
335         } else {
336                 gint last;
337
338                 alist = news_get_uncached_articles
339                         (session, item, 0, NULL, &last);
340                 news_delete_all_articles(item);
341                 item->last_num = last;
342         }
343
344         procmsg_set_flags(alist, item);
345
346         return alist;
347 }
348
349 gchar *news_fetch_msg(Folder *folder, FolderItem *item, gint num)
350 {
351         gchar *path, *filename;
352         NNTPSession *session;
353         gint ok;
354
355         g_return_val_if_fail(folder != NULL, NULL);
356         g_return_val_if_fail(item != NULL, NULL);
357
358         path = folder_item_get_path(item);
359         if (!is_dir_exist(path))
360                 make_dir_hier(path);
361         filename = g_strconcat(path, G_DIR_SEPARATOR_S, itos(num), NULL);
362         g_free(path);
363
364         if (is_file_exist(filename)) {
365                 debug_print("article %d has been already cached.\n", num);
366                 return filename;
367         }
368
369         session = news_session_get(folder);
370         if (!session) {
371                 g_free(filename);
372                 return NULL;
373         }
374
375         ok = news_select_group(session, item->path, NULL, NULL, NULL);
376         if (ok != NN_SUCCESS) {
377                 g_warning("can't select group %s\n", item->path);
378                 g_free(filename);
379                 return NULL;
380         }
381
382         debug_print("getting article %d...\n", num);
383         ok = news_get_article(NNTP_SESSION(REMOTE_FOLDER(folder)->session),
384                               num, filename);
385         if (ok < 0) {
386                 g_warning("can't read article %d\n", num);
387                 g_free(filename);
388                 return NULL;
389         }
390
391         return filename;
392 }
393
394 gint news_scan_group(Folder *folder, FolderItem *item)
395 {
396         NNTPSession *session;
397         gint num = 0, first = 0, last = 0;
398         gint ok;
399
400         g_return_val_if_fail(folder != NULL, -1);
401         g_return_val_if_fail(item != NULL, -1);
402
403         session = news_session_get(folder);
404         if (!session) return -1;
405
406         ok = news_select_group(session, item->path, &num, &first, &last);
407         if (ok != NN_SUCCESS) {
408                 log_warning("can't set group: %s\n", item->path);
409                 return -1;
410         }
411
412         if (num == 0) {
413                 item->new = item->unread = item->total = item->last_num = 0;
414                 return 0;
415         }
416
417 /*
418         path = folder_item_get_path(item);
419         if (path && is_dir_exist(path)) {
420                 procmsg_get_mark_sum(path, &new, &unread, &total, &min, &max,
421                                      first);
422         }
423         g_free(path);
424
425         if (max < first || last < min)
426                 new = unread = total = num;
427         else {
428                 if (min < first)
429                         min = first;
430
431                 if (last < max)
432                         max = last;
433                 else if (max < last) {
434                         new += last - max;
435                         unread += last - max;
436                 }
437
438                 if (new > num) new = num;
439                 if (unread > num) unread = num;
440         }
441
442         item->new = new;
443         item->unread = unread;
444         item->total = num;
445         item->last_num = last;
446 */
447         return 0;
448 }
449
450 static NewsGroupInfo *news_group_info_new(const gchar *name,
451                                           gint first, gint last, gchar type)
452 {
453         NewsGroupInfo *ginfo;
454
455         ginfo = g_new(NewsGroupInfo, 1);
456         ginfo->name = g_strdup(name);
457         ginfo->first = first;
458         ginfo->last = last;
459         ginfo->type = type;
460
461         return ginfo;
462 }
463
464 static void news_group_info_free(NewsGroupInfo *ginfo)
465 {
466         g_free(ginfo->name);
467         g_free(ginfo);
468 }
469
470 static gint news_group_info_compare(NewsGroupInfo *ginfo1,
471                                     NewsGroupInfo *ginfo2)
472 {
473         return g_strcasecmp(ginfo1->name, ginfo2->name);
474 }
475
476 GSList *news_get_group_list(Folder *folder)
477 {
478         gchar *path, *filename;
479         FILE *fp;
480         GSList *list = NULL;
481         GSList *last = NULL;
482         gchar buf[NNTPBUFSIZE];
483
484         g_return_val_if_fail(folder != NULL, NULL);
485         g_return_val_if_fail(folder->type == F_NEWS, NULL);
486
487         path = folder_item_get_path(FOLDER_ITEM(folder->node->data));
488         if (!is_dir_exist(path))
489                 make_dir_hier(path);
490         filename = g_strconcat(path, G_DIR_SEPARATOR_S, NEWSGROUP_LIST, NULL);
491         g_free(path);
492
493         if ((fp = fopen(filename, "rb")) == NULL) {
494                 NNTPSession *session;
495
496                 session = news_session_get(folder);
497                 if (!session) {
498                         g_free(filename);
499                         return NULL;
500                 }
501
502                 if (nntp_list(session->nntp_sock) != NN_SUCCESS) {
503                         g_free(filename);
504                         return NULL;
505                 }
506                 if (recv_write_to_file(SESSION(session)->sock, filename) < 0) {
507                         log_warning("can't retrieve newsgroup list\n");
508                         session_destroy(SESSION(session));
509                         REMOTE_FOLDER(folder)->session = NULL;
510                         g_free(filename);
511                         return NULL;
512                 }
513
514                 if ((fp = fopen(filename, "rb")) == NULL) {
515                         FILE_OP_ERROR(filename, "fopen");
516                         g_free(filename);
517                         return NULL;
518                 }
519         }
520
521         while (fgets(buf, sizeof(buf), fp) != NULL) {
522                 gchar *p = buf;
523                 gchar *name;
524                 gint last_num;
525                 gint first_num;
526                 gchar type;
527                 NewsGroupInfo *ginfo;
528
529                 p = strchr(p, ' ');
530                 if (!p) continue;
531                 *p = '\0';
532                 p++;
533                 name = buf;
534
535                 if (sscanf(p, "%d %d %c", &last_num, &first_num, &type) < 3)
536                         continue;
537
538                 ginfo = news_group_info_new(name, first_num, last_num, type);
539
540                 if (!last)
541                         last = list = g_slist_append(NULL, ginfo);
542                 else {
543                         last = g_slist_append(last, ginfo);
544                         last = last->next;
545                 }
546         }
547
548         fclose(fp);
549         g_free(filename);
550
551         list = g_slist_sort(list, (GCompareFunc)news_group_info_compare);
552
553         return list;
554 }
555
556 void news_group_list_free(GSList *group_list)
557 {
558         GSList *cur;
559
560         if (!group_list) return;
561
562         for (cur = group_list; cur != NULL; cur = cur->next)
563                 news_group_info_free((NewsGroupInfo *)cur->data);
564         g_slist_free(group_list);
565 }
566
567 void news_remove_group_list_cache(Folder *folder)
568 {
569         gchar *path, *filename;
570
571         g_return_if_fail(folder != NULL);
572         g_return_if_fail(folder->type == F_NEWS);
573
574         path = folder_item_get_path(FOLDER_ITEM(folder->node->data));
575         filename = g_strconcat(path, G_DIR_SEPARATOR_S, NEWSGROUP_LIST, NULL);
576         g_free(path);
577
578         if (is_file_exist(filename)) {
579                 if (remove(filename) < 0)
580                         FILE_OP_ERROR(filename, "remove");
581         }
582         g_free(filename);
583 }
584
585 gint news_post(Folder *folder, const gchar *file)
586 {
587         FILE *fp;
588         gint ok;
589
590         g_return_val_if_fail(folder != NULL, -1);
591         g_return_val_if_fail(folder->type == F_NEWS, -1);
592         g_return_val_if_fail(file != NULL, -1);
593
594         if ((fp = fopen(file, "rb")) == NULL) {
595                 FILE_OP_ERROR(file, "fopen");
596                 return -1;
597         }
598
599         ok = news_post_stream(folder, fp);
600
601         fclose(fp);
602
603         return ok;
604 }
605
606 gint news_post_stream(Folder *folder, FILE *fp)
607 {
608         NNTPSession *session;
609         gint ok;
610
611         g_return_val_if_fail(folder != NULL, -1);
612         g_return_val_if_fail(folder->type == F_NEWS, -1);
613         g_return_val_if_fail(fp != NULL, -1);
614
615         session = news_session_get(folder);
616         if (!session) return -1;
617
618         ok = nntp_post(session->nntp_sock, fp);
619         if (ok != NN_SUCCESS) {
620                 log_warning("can't post article.\n");
621                 return -1;
622         }
623
624         return 0;
625 }
626
627 static gint news_get_article_cmd(NNTPSession *session, const gchar *cmd,
628                                  gint num, gchar *filename)
629 {
630         gchar *msgid;
631
632         if (nntp_get_article(session->nntp_sock, cmd, num, &msgid)
633             != NN_SUCCESS)
634                 return -1;
635
636         debug_print("Message-Id = %s, num = %d\n", msgid, num);
637         g_free(msgid);
638
639         if (recv_write_to_file(session->nntp_sock->sock, filename) < 0) {
640                 log_warning("can't retrieve article %d\n", num);
641                 return -1;
642         }
643
644         return 0;
645 }
646
647 static gint news_remove_msg(Folder *folder, FolderItem *item, gint num)
648 {
649         gchar * dir;
650         gint r;
651
652         dir = folder_item_get_path(item);
653         debug_print("news_remove_msg: removing msg %d in %s\n",num,dir);
654         r = remove_numbered_files(dir, num, num);
655         g_free(dir);
656
657         return r;
658 }
659
660 static gint news_get_article(NNTPSession *session, gint num, gchar *filename)
661 {
662         return news_get_article_cmd(session, "ARTICLE", num, filename);
663 }
664
665 /**
666  * news_select_group:
667  * @session: Active NNTP session.
668  * @group: Newsgroup name.
669  * @num: Estimated number of articles.
670  * @first: First article number.
671  * @last: Last article number.
672  *
673  * Select newsgroup @group with the GROUP command if it is not already
674  * selected in @session, or article numbers need to be returned.
675  *
676  * Return value: NNTP result code.
677  **/
678 static gint news_select_group(NNTPSession *session, const gchar *group,
679                               gint *num, gint *first, gint *last)
680 {
681         gint ok;
682         gint num_, first_, last_;
683
684         if (!num || !first || !last) {
685                 if (session->group && g_strcasecmp(session->group, group) == 0)
686                         return NN_SUCCESS;
687                 num = &num_;
688                 first = &first_;
689                 last = &last_;
690         }
691
692         g_free(session->group);
693         session->group = NULL;
694
695         ok = nntp_group(session->nntp_sock, group, num, first, last);
696         if (ok == NN_SUCCESS)
697                 session->group = g_strdup(group);
698
699         return ok;
700 }
701
702 static GSList *news_get_uncached_articles(NNTPSession *session,
703                                           FolderItem *item, gint cache_last,
704                                           gint *rfirst, gint *rlast)
705 {
706         gint ok;
707         gint num = 0, first = 0, last = 0, begin = 0, end = 0;
708         gchar buf[NNTPBUFSIZE];
709         GSList *newlist = NULL;
710         GSList *llast = NULL;
711         MsgInfo *msginfo;
712
713         if (rfirst) *rfirst = -1;
714         if (rlast)  *rlast  = -1;
715
716         g_return_val_if_fail(session != NULL, NULL);
717         g_return_val_if_fail(item != NULL, NULL);
718         g_return_val_if_fail(item->folder != NULL, NULL);
719         g_return_val_if_fail(item->folder->type == F_NEWS, NULL);
720
721         ok = news_select_group(session, item->path, &num, &first, &last);
722         if (ok != NN_SUCCESS) {
723                 log_warning("can't set group: %s\n", item->path);
724                 return NULL;
725         }
726
727         /* calculate getting overview range */
728         if (first > last) {
729                 log_warning("invalid article range: %d - %d\n",
730                             first, last);
731                 return NULL;
732         }
733
734         if (rfirst) *rfirst = first;
735         if (rlast)  *rlast  = last;
736
737         if (cache_last < first)
738                 begin = first;
739         else if (last < cache_last)
740                 begin = first;
741         else if (last == cache_last) {
742                 debug_print("no new articles.\n");
743                 return NULL;
744         } else
745                 begin = cache_last + 1;
746         end = last;
747
748         if (prefs_common.max_articles > 0 &&
749             end - begin + 1 > prefs_common.max_articles)
750                 begin = end - prefs_common.max_articles + 1;
751
752         log_message(_("getting xover %d - %d in %s...\n"),
753                     begin, end, item->path);
754         if (nntp_xover(session->nntp_sock, begin, end) != NN_SUCCESS) {
755                 log_warning("can't get xover\n");
756                 return NULL;
757         }
758
759         for (;;) {
760                 if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
761                         log_warning("error occurred while getting xover.\n");
762                         return newlist;
763                 }
764
765                 if (buf[0] == '.' && buf[1] == '\r') break;
766
767                 msginfo = news_parse_xover(buf);
768                 if (!msginfo) {
769                         log_warning("invalid xover line: %s\n", buf);
770                         continue;
771                 }
772
773                 msginfo->folder = item;
774                 msginfo->flags.perm_flags = MSG_NEW|MSG_UNREAD;
775                 msginfo->flags.tmp_flags = MSG_NEWS;
776                 msginfo->newsgroups = g_strdup(item->path);
777
778                 if (!newlist)
779                         llast = newlist = g_slist_append(newlist, msginfo);
780                 else {
781                         llast = g_slist_append(llast, msginfo);
782                         llast = llast->next;
783                 }
784         }
785
786         if (nntp_xhdr(session->nntp_sock, "to", begin, end) != NN_SUCCESS) {
787                 log_warning("can't get xhdr\n");
788                 return newlist;
789         }
790
791         llast = newlist;
792
793         for (;;) {
794                 if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
795                         log_warning("error occurred while getting xhdr.\n");
796                         return newlist;
797                 }
798
799                 if (buf[0] == '.' && buf[1] == '\r') break;
800                 if (!llast) {
801                         g_warning("llast == NULL\n");
802                         continue;
803                 }
804
805                 msginfo = (MsgInfo *)llast->data;
806                 msginfo->to = news_parse_xhdr(buf, msginfo);
807
808                 llast = llast->next;
809         }
810
811         if (nntp_xhdr(session->nntp_sock, "cc", begin, end) != NN_SUCCESS) {
812                 log_warning("can't get xhdr\n");
813                 return newlist;
814         }
815
816         llast = newlist;
817
818         for (;;) {
819                 if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
820                         log_warning("error occurred while getting xhdr.\n");
821                         return newlist;
822                 }
823
824                 if (buf[0] == '.' && buf[1] == '\r') break;
825                 if (!llast) {
826                         g_warning("llast == NULL\n");
827                         continue;
828                 }
829
830                 msginfo = (MsgInfo *)llast->data;
831                 msginfo->cc = news_parse_xhdr(buf, msginfo);
832
833                 llast = llast->next;
834         }
835
836         return newlist;
837 }
838
839 #define PARSE_ONE_PARAM(p, srcp) \
840 { \
841         p = strchr(srcp, '\t'); \
842         if (!p) return NULL; \
843         else \
844                 *p++ = '\0'; \
845 }
846
847 static MsgInfo *news_parse_xover(const gchar *xover_str)
848 {
849         MsgInfo *msginfo;
850         gchar buf[NNTPBUFSIZE];
851         gchar *subject, *sender, *size, *line, *date, *msgid, *ref, *tmp, *xref;
852         gchar *p;
853         gint num, size_int, line_int;
854         gchar *xover_buf;
855
856         Xstrdup_a(xover_buf, xover_str, return NULL);
857
858         PARSE_ONE_PARAM(subject, xover_buf);
859         PARSE_ONE_PARAM(sender, subject);
860         PARSE_ONE_PARAM(date, sender);
861         PARSE_ONE_PARAM(msgid, date);
862         PARSE_ONE_PARAM(ref, msgid);
863         PARSE_ONE_PARAM(size, ref);
864         PARSE_ONE_PARAM(line, size);
865         PARSE_ONE_PARAM(xref, line);
866
867         tmp = strchr(xref, '\t');
868         if (!tmp) tmp = strchr(line, '\r');
869         if (!tmp) tmp = strchr(line, '\n');
870         if (tmp) *tmp = '\0';
871
872         num = atoi(xover_str);
873         size_int = atoi(size);
874         line_int = atoi(line);
875
876         /* set MsgInfo */
877         msginfo = procmsg_msginfo_new();
878         msginfo->msgnum = num;
879         msginfo->size = size_int;
880
881         msginfo->date = g_strdup(date);
882         msginfo->date_t = procheader_date_parse(NULL, date, 0);
883
884         conv_unmime_header(buf, sizeof(buf), sender, NULL);
885         msginfo->from = g_strdup(buf);
886         msginfo->fromname = procheader_get_fromname(buf);
887
888         conv_unmime_header(buf, sizeof(buf), subject, NULL);
889         msginfo->subject = g_strdup(buf);
890
891         extract_parenthesis(msgid, '<', '>');
892         remove_space(msgid);
893         if (*msgid != '\0')
894                 msginfo->msgid = g_strdup(msgid);
895
896         msginfo->references = g_strdup(ref);
897         eliminate_parenthesis(ref, '(', ')');
898         if ((p = strrchr(ref, '<')) != NULL) {
899                 extract_parenthesis(p, '<', '>');
900                 remove_space(p);
901                 if (*p != '\0')
902                         msginfo->inreplyto = g_strdup(p);
903         }
904
905         msginfo->xref = g_strdup(xref);
906         p = msginfo->xref+strlen(msginfo->xref) - 1;
907         while (*p == '\r' || *p == '\n') {
908                 *p = '\0';
909                 p--;
910         }
911
912         return msginfo;
913 }
914
915 static gchar *news_parse_xhdr(const gchar *xhdr_str, MsgInfo *msginfo)
916 {
917         gchar *p;
918         gchar *tmp;
919         gint num;
920
921         p = strchr(xhdr_str, ' ');
922         if (!p)
923                 return NULL;
924         else
925                 p++;
926
927         num = atoi(xhdr_str);
928         if (msginfo->msgnum != num) return NULL;
929
930         tmp = strchr(p, '\r');
931         if (!tmp) tmp = strchr(p, '\n');
932
933         if (tmp)
934                 return g_strndup(p, tmp - p);
935         else
936                 return g_strdup(p);
937 }
938
939 static GSList *news_delete_old_articles(GSList *alist, FolderItem *item,
940                                         gint first)
941 {
942         GSList *cur, *next;
943         MsgInfo *msginfo;
944         gchar *dir;
945
946         g_return_val_if_fail(item != NULL, alist);
947         g_return_val_if_fail(item->folder != NULL, alist);
948         g_return_val_if_fail(item->folder->type == F_NEWS, alist);
949
950         if (first < 2) return alist;
951
952         debug_print("Deleting cached articles 1 - %d ...\n", first - 1);
953
954         dir = folder_item_get_path(item);
955         remove_numbered_files(dir, 1, first - 1);
956         g_free(dir);
957
958         for (cur = alist; cur != NULL; ) {
959                 next = cur->next;
960
961                 msginfo = (MsgInfo *)cur->data;
962                 if (msginfo && msginfo->msgnum < first) {
963                         procmsg_msginfo_free(msginfo);
964                         alist = g_slist_remove(alist, msginfo);
965                 }
966
967                 cur = next;
968         }
969
970         return alist;
971 }
972
973 static void news_delete_all_articles(FolderItem *item)
974 {
975         gchar *dir;
976
977         g_return_if_fail(item != NULL);
978         g_return_if_fail(item->folder != NULL);
979         g_return_if_fail(item->folder->type == F_NEWS);
980
981         debug_print("Deleting all cached articles...\n");
982
983         dir = folder_item_get_path(item);
984         remove_all_numbered_files(dir);
985         g_free(dir);
986 }
987
988 static void news_delete_expired_caches(GSList *alist, FolderItem *item)
989 {
990         gchar *dir;
991
992         g_return_if_fail(item != NULL);
993         g_return_if_fail(item->folder != NULL);
994         g_return_if_fail(item->folder->type == F_NEWS);
995
996         debug_print("Deleting expired cached articles...\n");
997
998         dir = folder_item_get_path(item);
999         remove_expired_files(dir, 24 * 7);
1000         g_free(dir);
1001 }
1002
1003 gint news_cancel_article(Folder * folder, MsgInfo * msginfo)
1004 {
1005         gchar * tmp;
1006         FILE * tmpfp;
1007         gchar buf[BUFFSIZE];
1008
1009         tmp = g_strdup_printf("%s%ctmp%d", g_get_tmp_dir(),
1010                               G_DIR_SEPARATOR, (gint)msginfo);
1011         if (tmp == NULL)
1012                 return -1;
1013
1014         if ((tmpfp = fopen(tmp, "wb")) == NULL) {
1015                 FILE_OP_ERROR(tmp, "fopen");
1016                 return -1;
1017         }
1018         if (change_file_mode_rw(tmpfp, tmp) < 0) {
1019                 FILE_OP_ERROR(tmp, "chmod");
1020                 g_warning("can't change file mode\n");
1021         }
1022         
1023         fprintf(tmpfp, "From: %s\r\n", msginfo->from);
1024         fprintf(tmpfp, "Newsgroups: %s\r\n", msginfo->newsgroups);
1025         fprintf(tmpfp, "Subject: cmsg cancel <%s>\r\n", msginfo->msgid);
1026         fprintf(tmpfp, "Control: cancel <%s>\r\n", msginfo->msgid);
1027         fprintf(tmpfp, "Approved: %s\r\n", msginfo->from);
1028         fprintf(tmpfp, "X-Cancelled-by: %s\r\n", msginfo->from);
1029         get_rfc822_date(buf, sizeof(buf));
1030         fprintf(tmpfp, "Date: %s\r\n", buf);
1031         fprintf(tmpfp, "\r\n");
1032         fprintf(tmpfp, "removed with sylpheed\r\n");
1033
1034         fclose(tmpfp);
1035
1036         news_post(folder, tmp);
1037         remove(tmp);
1038
1039         g_free(tmp);
1040
1041         return 0;
1042 }
1043
1044 gint news_get_num_list(Folder *folder, FolderItem *item, GSList **msgnum_list)
1045 {
1046         NNTPSession *session;
1047         gint i, ok, num, first, last, nummsgs = 0;
1048         gchar *dir;
1049
1050         g_return_val_if_fail(item != NULL, -1);
1051         g_return_val_if_fail(item->folder != NULL, -1);
1052         g_return_val_if_fail(item->folder->type == F_NEWS, -1);
1053
1054         session = news_session_get(folder);
1055         g_return_val_if_fail(session != NULL, -1);
1056
1057         ok = news_select_group(session, item->path, &num, &first, &last);
1058         if (ok != NN_SUCCESS) {
1059                 log_warning(_("can't set group: %s\n"), item->path);
1060                 return -1;
1061         }
1062
1063         if(last < first) {
1064                 log_warning(_("invalid article range: %d - %d\n"),
1065                             first, last);
1066                 return 0;
1067         }
1068
1069         for(i = first; i <= last; i++) {
1070                 *msgnum_list = g_slist_prepend(*msgnum_list, GINT_TO_POINTER(i));
1071                 nummsgs++;
1072         }
1073
1074         dir = folder_item_get_path(item);
1075         debug_print("removing old messages from %d to %d in %s\n", first, last, dir);
1076         remove_numbered_files(dir, 1, first - 1);
1077         g_free(dir);
1078
1079         return nummsgs;
1080 }
1081
1082 #define READ_TO_LISTEND(hdr) \
1083         while (!(buf[0] == '.' && buf[1] == '\r')) { \
1084                 if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) { \
1085                         log_warning(_("error occurred while getting %s.\n"), hdr); \
1086                         return msginfo; \
1087                 } \
1088         }
1089
1090 MsgInfo *news_get_msginfo(Folder *folder, FolderItem *item, gint num)
1091 {
1092         NNTPSession *session;
1093         MsgInfo *msginfo = NULL;
1094         gchar buf[NNTPBUFSIZE];
1095
1096         session = news_session_get(folder);
1097         g_return_val_if_fail(session != NULL, NULL);
1098         g_return_val_if_fail(item != NULL, NULL);
1099         g_return_val_if_fail(item->folder != NULL, NULL);
1100         g_return_val_if_fail(item->folder->type == F_NEWS, NULL);
1101
1102         log_message(_("getting xover %d in %s...\n"),
1103                     num, item->path);
1104         if (nntp_xover(session->nntp_sock, num, num) != NN_SUCCESS) {
1105                 log_warning(_("can't get xover\n"));
1106                 return NULL;
1107         }
1108         
1109         if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
1110                 log_warning(_("error occurred while getting xover.\n"));
1111                 return NULL;
1112         }
1113         
1114         msginfo = news_parse_xover(buf);
1115         if (!msginfo) {
1116                 log_warning(_("invalid xover line: %s\n"), buf);
1117         }
1118
1119         READ_TO_LISTEND("xover");
1120
1121         if(!msginfo)
1122                 return NULL;
1123         
1124         msginfo->folder = item;
1125         msginfo->flags.perm_flags = MSG_NEW|MSG_UNREAD;
1126         msginfo->flags.tmp_flags = MSG_NEWS;
1127         msginfo->newsgroups = g_strdup(item->path);
1128
1129         if (nntp_xhdr(session->nntp_sock, "to", num, num) != NN_SUCCESS) {
1130                 log_warning(_("can't get xhdr\n"));
1131                 return msginfo;
1132         }
1133
1134         if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
1135                 log_warning(_("error occurred while getting xhdr.\n"));
1136                 return msginfo;
1137         }
1138
1139         msginfo->to = news_parse_xhdr(buf, msginfo);
1140
1141         READ_TO_LISTEND("xhdr (to)");
1142
1143         if (nntp_xhdr(session->nntp_sock, "cc", num, num) != NN_SUCCESS) {
1144                 log_warning(_("can't get xhdr\n"));
1145                 return msginfo;
1146         }
1147
1148         if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
1149                 log_warning(_("error occurred while getting xhdr.\n"));
1150                 return msginfo;
1151         }
1152
1153         msginfo->cc = news_parse_xhdr(buf, msginfo);
1154
1155         READ_TO_LISTEND("xhdr (cc)");
1156
1157         return msginfo;
1158 }
1159
1160 /*!
1161  *\brief        Set/reset/update progressbar, max. 10 redraws/second
1162  *
1163  *\param        action What to do with the statusbar
1164  *                      = 0 : Reset to zero
1165  *                      < 0 : Init maximum to -(action)
1166  *                      > 0 : Increase by (action)
1167  */
1168 void news_doprogress(glong action) {
1169         static glong curcount=0;
1170         static glong maxval;
1171         static MainWindow *mainwin=0;
1172         static struct timeval tv_prev, tv_cur;
1173         
1174         if (!mainwin)
1175                 mainwin = mainwindow_get_mainwindow();
1176         g_return_if_fail(mainwin);
1177
1178         if (action < 0)
1179                 maxval = -action;
1180         else if (action == 0) {
1181                 curcount=0;
1182                 gtk_progress_bar_update(
1183                         GTK_PROGRESS_BAR(mainwin->progressbar), 0.0);
1184         } else {
1185                 curcount += action;
1186                 gettimeofday(&tv_cur, NULL);
1187                 if (!(tv_cur.tv_sec - tv_prev.tv_sec > 0 ||
1188                         tv_cur.tv_usec - tv_prev.tv_usec > 100)
1189                     || !maxval)
1190                         return;
1191
1192                 gtk_progress_bar_update(
1193                         GTK_PROGRESS_BAR(mainwin->progressbar),
1194                         (float)curcount/maxval);
1195                 while (gtk_events_pending()) gtk_main_iteration ();
1196                 gettimeofday(&tv_prev, NULL);
1197         }
1198 }
1199
1200 static GSList *news_get_msginfos_for_range(NNTPSession *session, FolderItem *item, guint begin, guint end)
1201 {
1202         gchar buf[NNTPBUFSIZE];
1203         GSList *newlist = NULL;
1204         GSList *llast = NULL;
1205         MsgInfo *msginfo;
1206
1207         g_return_val_if_fail(session != NULL, NULL);
1208         g_return_val_if_fail(item != NULL, NULL);
1209
1210         inc_lock();
1211         log_message(_("getting xover %d - %d in %s...\n"),
1212                     begin, end, item->path);
1213         if (nntp_xover(session->nntp_sock, begin, end) != NN_SUCCESS) {
1214                 log_warning(_("can't get xover\n"));
1215                 goto unlock_leave;
1216         }
1217
1218         news_doprogress(-3*(end-begin));
1219         statusbar_print_all(_("Getting overview (%d articles)..."),end-begin);
1220         for (;;) {
1221                 if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
1222                         log_warning(_("error occurred while getting xover.\n"));
1223                         goto unlock_leave;
1224                 }
1225                 news_doprogress(1);
1226
1227                 if (buf[0] == '.' && buf[1] == '\r') break;
1228
1229                 msginfo = news_parse_xover(buf);
1230                 if (!msginfo) {
1231                         log_warning(_("invalid xover line: %s\n"), buf);
1232                         continue;
1233                 }
1234
1235                 msginfo->folder = item;
1236                 msginfo->flags.perm_flags = MSG_NEW|MSG_UNREAD;
1237                 msginfo->flags.tmp_flags = MSG_NEWS;
1238                 msginfo->newsgroups = g_strdup(item->path);
1239
1240                 if (!newlist)
1241                         llast = newlist = g_slist_append(newlist, msginfo);
1242                 else {
1243                         llast = g_slist_append(llast, msginfo);
1244                         llast = llast->next;
1245                 }
1246         }
1247
1248         if (nntp_xhdr(session->nntp_sock, "to", begin, end) != NN_SUCCESS) {
1249                 log_warning(_("can't get xhdr\n"));
1250                         goto unlock_leave;
1251         }
1252
1253         llast = newlist;
1254
1255         statusbar_pop_all();
1256         statusbar_print_all(_("Getting headers (%d articles)..."),end-begin);
1257         for (;;) {
1258                 if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
1259                         log_warning(_("error occurred while getting xhdr.\n"));
1260                         goto unlock_leave;
1261                 }
1262                 news_doprogress(1);
1263
1264                 if (buf[0] == '.' && buf[1] == '\r') break;
1265                 if (!llast) {
1266                         g_warning("llast == NULL\n");
1267                         continue;
1268                 }
1269
1270                 msginfo = (MsgInfo *)llast->data;
1271                 msginfo->to = news_parse_xhdr(buf, msginfo);
1272
1273                 llast = llast->next;
1274         }
1275
1276         if (nntp_xhdr(session->nntp_sock, "cc", begin, end) != NN_SUCCESS) {
1277                 log_warning(_("can't get xhdr\n"));
1278                 goto unlock_leave;
1279         }
1280
1281         llast = newlist;
1282
1283         statusbar_pop_all();
1284         statusbar_print_all(_("Getting crossposted articles (%d messages)..."),end-begin);
1285         for (;;) {
1286                 if (sock_gets(SESSION(session)->sock, buf, sizeof(buf)) < 0) {
1287                         log_warning(_("error occurred while getting xhdr.\n"));
1288                         goto unlock_leave;
1289                 }
1290                 news_doprogress(1);
1291
1292                 if (buf[0] == '.' && buf[1] == '\r') break;
1293                 if (!llast) {
1294                         g_warning("llast == NULL\n");
1295                         continue;
1296                 }
1297
1298                 msginfo = (MsgInfo *)llast->data;
1299                 msginfo->cc = news_parse_xhdr(buf, msginfo);
1300
1301                 llast = llast->next;
1302         }
1303
1304 unlock_leave:
1305         news_doprogress(0);
1306         statusbar_pop_all();
1307         inc_unlock();
1308         return newlist;
1309 }
1310
1311 gint news_fetch_msgnum_sort(gconstpointer a, gconstpointer b)
1312 {
1313         return (GPOINTER_TO_INT(a) - GPOINTER_TO_INT(b));
1314 }
1315
1316 GSList *news_get_msginfos(Folder *folder, FolderItem *item, GSList *msgnum_list)
1317 {
1318         NNTPSession *session;
1319         GSList *elem, *msginfo_list = NULL, *tmp_msgnum_list, *tmp_msginfo_list;
1320         guint first, last, next;
1321         
1322         g_return_val_if_fail(folder != NULL, NULL);
1323         g_return_val_if_fail(folder->type == F_NEWS, NULL);
1324         g_return_val_if_fail(msgnum_list != NULL, NULL);
1325         g_return_val_if_fail(item != NULL, NULL);
1326         
1327         session = news_session_get(folder);
1328         g_return_val_if_fail(session != NULL, NULL);
1329
1330         tmp_msgnum_list = g_slist_copy(msgnum_list);
1331         tmp_msgnum_list = g_slist_sort(tmp_msgnum_list, news_fetch_msgnum_sort);
1332
1333         first = GPOINTER_TO_INT(tmp_msgnum_list->data);
1334         last = first;
1335         for(elem = g_slist_next(tmp_msgnum_list); elem != NULL; elem = g_slist_next(elem)) {
1336                 next = GPOINTER_TO_INT(elem->data);
1337                 if(next != (last + 1)) {
1338                         tmp_msginfo_list = news_get_msginfos_for_range(session, item, first, last);
1339                         msginfo_list = g_slist_concat(msginfo_list, tmp_msginfo_list);
1340                         first = next;
1341                 }
1342                 last = next;
1343         }
1344         tmp_msginfo_list = news_get_msginfos_for_range(session, item, first, last);
1345         msginfo_list = g_slist_concat(msginfo_list, tmp_msginfo_list);
1346
1347         g_slist_free(tmp_msgnum_list);
1348         
1349         return msginfo_list;
1350 }