src/plugins/rssyl/rssyl_add_item.c

   1 /*
   2  * Claws-Mail-- a GTK+ based, lightweight, and fast e-mail client
   3  * Copyright (C) 1999-2004 Hiroyuki Yamamoto
   4  * This file (C) 2005 Andrej Kacian <andrej@kacian.sk>
   5  *
   6  * - DESCRIPTION HERE
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  21  */
  22
  23 #ifdef HAVE_CONFIG_H
  24 #  include "config.h"
  25 #endif
  26
  27 /* Global includes */
  28 #include <errno.h>
  29 #include <glib.h>
  30 #include <glib/gi18n.h>
  31 #include <string.h>
  32
  33 /* Claws Mail includes */
  34 #include <codeconv.h>
  35 #include <procmsg.h>
  36 #include <common/utils.h>
  37
  38 /* Local includes */
  39 #include "libfeed/date.h"
  40 #include "libfeed/feeditem.h"
  41 #include "parse822.h"
  42 #include "rssyl.h"
  43 #include "rssyl_deleted.h"
  44 #include "rssyl_feed.h"
  45 #include "rssyl_parse_feed.h"
  46 #include "strutils.h"
  47
  48 /* rssyl_cb_feed_compare()
  49  *
  50  * GCompareFunc function called by glib2's g_slist_find_custom().
  51  */
  52
  53 static gint rssyl_cb_feed_compare(const FeedItem *a, const FeedItem *b)
  54 {
  55         gboolean date_eq = FALSE, url_eq = FALSE, title_eq = FALSE;
  56         gboolean pubdate_eq = FALSE, moddate_eq = FALSE;
  57         gboolean no_url = FALSE, no_date = FALSE, no_title = FALSE;
  58         gboolean no_pubdate = FALSE, no_moddate = FALSE;
  59         gchar *atit = NULL, *btit = NULL;
  60
  61         g_return_val_if_fail(a != NULL && b != NULL, 1);
  62
  63         /* ID should be unique. If it matches, we've found what we came for. */
  64         if( (a->id != NULL) && (b->id != NULL) ) {
  65                         if( !strcmp(a->id, b->id) ) {
  66                                 return 0;
  67                         }
  68
  69                         /* If both IDs are present, but they do not match, these are not the
  70                          * droids we're looking for. */
  71                         return 1;
  72         }
  73
  74         /* Ok, we have no ID to aid us. Let's have a look at item timestamps
  75          * and item title & url. */
  76         if( (a->url != NULL) && (b->url != NULL) ) {
  77                 if( !strcmp(a->url, b->url) )
  78                         url_eq = TRUE;
  79         } else
  80                 no_url = TRUE;
  81
  82         /* Now we prepare some boolean flags to help us express comparing choices
  83          * later on. */
  84
  85         /* Title */
  86         if( (a->title != NULL) && (b->title != NULL) ) {
  87                 atit = conv_unmime_header(a->title, CS_UTF_8, FALSE);
  88                 btit = conv_unmime_header(b->title, CS_UTF_8, FALSE);
  89                 if( !strcmp(atit, btit) )
  90                         title_eq = TRUE;
  91                 g_free(atit);
  92                 g_free(btit);
  93         } else
  94                 no_title = TRUE;
  95
  96         /* Published date */
  97         if (b->date_published <= 0) {
  98                 no_pubdate = TRUE;
  99         } else {
 100                 if (a->date_published == b->date_published)
 101                         pubdate_eq = TRUE;
 102         }
 103
 104         /* Modified date */
 105         if (b->date_modified <= 0) {
 106                 no_moddate = TRUE;
 107         } else {
 108                 if (a->date_modified == b->date_modified)
 109                         moddate_eq = TRUE;
 110         }
 111
 112         if (no_pubdate && no_moddate)
 113                 no_date = TRUE;
 114
 115         if (pubdate_eq || (no_pubdate && moddate_eq))
 116                 date_eq = TRUE;
 117
 118         /* If timestamp and url match, it is reasonable to assume
 119          * we found our item. */
 120         if (url_eq && date_eq)
 121                 return 0;
 122
 123         /* Likewise if timestamp and title match. */
 124         if (title_eq && date_eq)
 125                 return 0;
 126
 127         /* There is no timestamp and the url matches (or there is none),
 128          * we need to compare titles, ... */
 129         if( (no_url || url_eq) && no_date ) {
 130                 if( title_eq )
 131                         return 0;
 132                 else
 133                         return 1;
 134         }
 135
 136         /* ... and as a last resort, if there is no title, item texts. */
 137         if( no_title && a->text && b->text ) {
 138                 if( !strcmp(a->text, b->text) )
 139                         return 0;
 140                 else
 141                         return 1;
 142         }
 143
 144         /* We don't know this item. */
 145         return 1;
 146 }
 147
 148 enum {
 149         ITEM_UNCHANGED,
 150         ITEM_CHANGED_TEXTONLY,
 151         ITEM_CHANGED
 152 };
 153
 154 static gint rssyl_feed_item_changed(FeedItem *new_item, FeedItem *old_item )
 155 {
 156         debug_print("RSSyl: comparing '%s' and '%s'\n",
 157                         new_item->title, old_item->title);
 158
 159         /* if both have title ... */
 160         if( old_item->title && new_item->title ) {
 161                 gchar *old = conv_unmime_header(old_item->title, CS_UTF_8, FALSE);
 162                 gchar *new = conv_unmime_header(new_item->title, CS_UTF_8, FALSE);
 163                 if( strcmp(old, new) != 0 ) { /* ... compare "unmimed" titles */
 164                         debug_print("RSSyl:\t\titem titles differ:\nOLD: '%s'\nNEW: '%s'\n",
 165                                         old, new);
 166                         g_free(old);
 167                         g_free(new);
 168                         return ITEM_CHANGED;
 169                 }
 170                 g_free(old);
 171                 g_free(new);
 172         } else {
 173                 /* if atleast one has a title, they differ */
 174                 if( old_item->title || new_item->title ) {
 175                         debug_print("RSSyl:\t\t+/- title\n");
 176                         return ITEM_CHANGED;
 177                 }
 178         }
 179
 180         if( old_item->author && new_item->author ) {
 181                 gchar *old = conv_unmime_header(old_item->author, CS_UTF_8, TRUE);
 182                 gchar *new = conv_unmime_header(new_item->author, CS_UTF_8, TRUE);
 183                 if( strcmp(old, new) ) {  /* ... compare "unmimed" authors */
 184                         g_free(old);
 185                         g_free(new);
 186                         debug_print("RSSyl:\t\titem authors differ\n");
 187                         return ITEM_CHANGED;
 188                 }
 189                 g_free(old);
 190                 g_free(new);
 191         } else {
 192                 /* if atleast one has author, they differ */
 193                 if( old_item->author || new_item->author ) {
 194                         debug_print("RSSyl:\t\t+/- author\n");
 195                         return ITEM_CHANGED;
 196                 }
 197         }
 198
 199         /* if both have text ... */
 200         if( old_item->text && new_item->text ) {
 201                 if( strcmp(old_item->text, new_item->text) ) { /* ... compare them */
 202                         debug_print("RSSyl:\t\titem texts differ\n");
 203                         debug_print("\nOLD: '%s'\n", old_item->text);
 204                         debug_print("\nNEW: '%s'\n", new_item->text);
 205
 206                         return ITEM_CHANGED_TEXTONLY;
 207                 }
 208         } else {
 209                 /* if at least one has some text, they differ */
 210                 if( old_item->text || new_item->text ) {
 211                         debug_print("RSSyl:\t\t+/- text\n");
 212                         return ITEM_CHANGED_TEXTONLY;
 213                 }
 214         }
 215
 216         /* they don't seem to differ */
 217         return ITEM_UNCHANGED;
 218 }
 219
 220 enum {
 221         EXISTS_NEW,
 222         EXISTS_UNCHANGED,
 223         EXISTS_CHANGED,
 224         EXISTS_CHANGED_TEXTONLY
 225 };
 226
 227 /* rssyl_feed_item_exists()
 228  *
 229  * Returns 1 if a feed item already exists locally, 2 if there's a changed
 230  * item with link that already belongs to existing item, 3 if only item's
 231  * text has changed, 0 if item is new.
 232  */
 233
 234 static guint rssyl_feed_item_exists(RFolderItem *ritem, FeedItem *fitem,
 235                 FeedItem **oldfitem)
 236 {
 237         GSList *item = NULL;
 238         FeedItem *efitem = NULL;
 239         gint changed;
 240
 241         g_return_val_if_fail(ritem != NULL, FALSE);
 242         g_return_val_if_fail(fitem != NULL, FALSE);
 243
 244         if( ritem->items == NULL || g_slist_length(ritem->items) == 0 )
 245                 return EXISTS_NEW;
 246
 247         if( (item = g_slist_find_custom(ritem->items,
 248                                         (gconstpointer)fitem, (GCompareFunc)rssyl_cb_feed_compare)) ) {
 249                 efitem = (FeedItem *)item->data;
 250                 if( (changed = rssyl_feed_item_changed(fitem, efitem)) > ITEM_UNCHANGED ) {
 251                         *oldfitem = efitem;
 252                         if (changed == ITEM_CHANGED_TEXTONLY)
 253                                 return EXISTS_CHANGED_TEXTONLY;
 254                         else
 255                                 return EXISTS_CHANGED;
 256                 }
 257
 258                 return EXISTS_UNCHANGED;
 259         }
 260
 261         return EXISTS_NEW;
 262 }
 263
 264 /* =============================================================== */
 265
 266 void rssyl_add_item(RFolderItem *ritem, FeedItem *feed_item)
 267 {
 268         FeedItem *old_item = NULL;
 269         MsgFlags *flags;
 270         MsgPermFlags oldperm_flags = 0;
 271         MsgInfo *msginfo;
 272         FILE *f;
 273         gint fd, d, dif;
 274         time_t tmpd;
 275         gchar *meta_charset = NULL;
 276         gchar *baseurl = NULL;
 277         gchar *template = NULL;
 278         gchar *tmp = NULL, *tmpurl = NULL, *tmpid = NULL;
 279         gchar *dirname = NULL;
 280         gchar *text = NULL;
 281         gchar *heading = NULL;
 282         gchar *pathbasename = NULL;
 283         gchar hdr[1024];
 284         FeedItemEnclosure *enc = NULL;
 285         RFeedCtx *ctx;
 286
 287         g_return_if_fail(ritem != NULL);
 288
 289         /* If item title is empty, try to fill it from source title (Atom only). */
 290         tmp = feed_item_get_sourcetitle(feed_item);
 291         if( feed_item_get_title(feed_item) == NULL ||
 292                         strlen(feed_item->title) == 0 ) {
 293                 if( tmp != NULL && strlen(tmp) > 0 )
 294                         feed_item_set_title(feed_item, tmp);
 295                 else
 296                         feed_item_set_title(feed_item, C_("Empty RSS feed title placeholder", "(empty)"));
 297         }
 298
 299 /*
 300         if (feed_item_get_id(feed_item) == NULL) {
 301                 debug_print("RSSyl: item ID empty, using its URL as ID.\n");
 302                 feed_item_set_id(feed_item, feed_item_get_url(feed_item));
 303         }
 304 */
 305
 306         /* If neither item date is set, use date from source (Atom only). */
 307         if( feed_item_get_date_modified(feed_item) == -1 &&
 308                         feed_item_get_date_published(feed_item) == -1 )
 309                 feed_item_set_date_published(feed_item,
 310                                 feed_item_get_sourcedate(feed_item));
 311
 312         /* Fix up subject, url and ID (rssyl_format_string()) so that
 313          * comparing doesn't break. */
 314         debug_print("RSSyl: fixing up subject '%s'\n", feed_item_get_title(feed_item));
 315         tmp = rssyl_format_string(feed_item_get_title(feed_item), TRUE, TRUE);
 316         feed_item_set_title(feed_item, tmp);
 317         g_free(tmp);
 318         debug_print("RSSyl: fixing up URL\n");
 319         tmp = rssyl_format_string(feed_item_get_url(feed_item), FALSE, TRUE);
 320         feed_item_set_url(feed_item, tmp);
 321         g_free(tmp);
 322         if( feed_item_get_id(feed_item) != NULL ) {
 323                 debug_print("RSSyl: fixing up ID\n");
 324                 tmp = rssyl_format_string(feed_item_get_id(feed_item), FALSE, TRUE);
 325                 feed_item_set_id(feed_item, tmp);
 326                 g_free(tmp);
 327         }
 328
 329         /* If there's a summary, but no text, use summary as text. */
 330         if( feed_item_get_text(feed_item) == NULL &&
 331                         (tmp = feed_item_get_summary(feed_item)) != NULL ) {
 332                 feed_item_set_text(feed_item, tmp);
 333                 g_free(feed_item->summary);     /* We do not need summary in rssyl now. */
 334                 feed_item->summary = NULL;
 335         }
 336
 337         /* Do not add if the item already exists, update if it does exist, but
 338          * has changed. */
 339         dif = rssyl_feed_item_exists(ritem, feed_item, &old_item);
 340         debug_print("RSSyl: rssyl_feed_item_exists returned %d\n", dif);
 341
 342         if( dif == EXISTS_UNCHANGED ) {
 343                 debug_print("RSSyl: This item already exists, skipping...\n");
 344                 return;
 345         }
 346
 347         /* Item is already in the list, but has changed */
 348         if( dif >= EXISTS_CHANGED && old_item != NULL ) {
 349                 debug_print("RSSyl: Item changed, removing old one and adding new.\n");
 350
 351                 /* Store permflags of the old item. */
 352                 ctx = (RFeedCtx *)old_item->data;
 353                 pathbasename = g_path_get_basename(ctx->path);
 354                 msginfo = folder_item_get_msginfo((FolderItem *)ritem,
 355                                                 atoi(pathbasename));
 356                 g_free(pathbasename);
 357                 oldperm_flags = msginfo->flags.perm_flags;
 358
 359                 ritem->items = g_slist_remove(ritem->items, old_item);
 360                 if (g_unlink(ctx->path) != 0) {
 361                         debug_print("RSSyl: Error, could not delete file '%s': %s\n",
 362                                         ctx->path, g_strerror(errno));
 363                 }
 364
 365                 g_free(ctx->path);
 366                 feed_item_free(old_item);
 367                 old_item = NULL;
 368         }
 369
 370         /* Check against list of deleted items. */
 371         if (rssyl_deleted_check(ritem->deleted_items, feed_item)) {
 372                 debug_print("RSSyl: Item '%s' found among deleted items, NOT adding it.\n",
 373                                 feed_item_get_title(feed_item));
 374                 return;
 375         }
 376
 377         /* Add a new item, formatting its title along the way */
 378         debug_print("RSSyl: Adding item '%s'\n", feed_item_get_title(feed_item));
 379         ritem->items = g_slist_prepend(ritem->items, feed_item_copy(feed_item));
 380
 381         dirname = folder_item_get_path(&ritem->item);
 382         template = g_strconcat(dirname, G_DIR_SEPARATOR_S,
 383                         RSSYL_TMP_TEMPLATE, NULL);
 384         if ((fd = g_mkstemp(template)) < 0) {
 385                 g_warning("Couldn't g_mkstemp('%s'), not adding message!", template);
 386                 g_free(dirname);
 387                 g_free(template);
 388                 return;
 389         }
 390
 391         f = fdopen(fd, "w");
 392         if (f == NULL) {
 393                 g_warning("Couldn't open file '%s', not adding message!", template);
 394                 g_free(dirname);
 395                 g_free(template);
 396                 return;
 397         }
 398
 399         /* From */
 400         if( (tmp = feed_item_get_author(feed_item)) != NULL ) {
 401                 if( g_utf8_validate(tmp, -1, NULL)) {
 402                         conv_encode_header_full(hdr, 1023, tmp, strlen("From: "),
 403                                         TRUE, CS_UTF_8);
 404                         fprintf(f, "From: %s\n", hdr);
 405                 } else
 406                         fprintf(f, "From: %s\n", tmp);
 407         }
 408
 409         /* Date */
 410         if( (tmpd = feed_item_get_date_modified(feed_item)) != -1 ) {
 411                 tmp = createRFC822Date(&tmpd);
 412                 debug_print("RSSyl: using date_modified: '%s'\n", tmp);
 413         } else if( (tmpd = feed_item_get_date_published(feed_item)) != -1 ) {
 414                 tmp = createRFC822Date(&tmpd);
 415                 debug_print("RSSyl: using date_published: '%s'\n", tmp);
 416         } else {
 417                 tmpd = time(NULL);
 418                 tmp = createRFC822Date(&tmpd);
 419         }
 420
 421         if( tmp != NULL ) {
 422                 fprintf(f, "Date: %s\n", tmp);
 423                 g_free(tmp);
 424         }
 425
 426         if( (tmp = feed_item_get_title(feed_item)) != NULL ) {
 427
 428                 /* (Atom only) Strip HTML markup from title for the Subject line. */
 429                 if( feed_item_get_title_format(feed_item) == FEED_ITEM_TITLE_HTML
 430                                 || feed_item_get_title_format(feed_item) == FEED_ITEM_TITLE_XHTML) {
 431                         debug_print("RSSyl: item title is HTML/XHTML, stripping tags for Subject line\n");
 432                         tmp = g_strdup(tmp);
 433                         strip_html(tmp);
 434                 }
 435
 436                 if( g_utf8_validate(tmp, -1, NULL) ) {
 437                         conv_encode_header_full(hdr, 1023, tmp, strlen("Subject: "),
 438                                         FALSE, CS_UTF_8);
 439                         debug_print("RSSyl: Subject: %s\n", hdr);
 440                         fprintf(f, "Subject: %s\n", hdr);
 441                 } else
 442                         fprintf(f, "Subject: %s\n", tmp);
 443
 444                 if( feed_item_get_title_format(feed_item) == FEED_ITEM_TITLE_HTML
 445                                 || feed_item_get_title_format(feed_item) == FEED_ITEM_TITLE_XHTML) {
 446                         g_free(tmp);
 447                         fprintf(f, "X-RSSyl-OrigTitle: %s\n", feed_item_get_title(feed_item));
 448                 }
 449         } else {
 450                 debug_print("RSSyl: No feed title, it seems\n");
 451                 fprintf(f, "Subject: (empty)\n");
 452         }
 453
 454         /* X-RSSyl-URL */
 455         if( (tmpurl = feed_item_get_url(feed_item)) == NULL ) {
 456                 if( feed_item_get_id(feed_item) != NULL &&
 457                                 feed_item_id_is_permalink(feed_item) ) {
 458                         tmpurl = feed_item_get_id(feed_item);
 459                 }
 460         }
 461
 462         if( tmpurl != NULL )
 463                 fprintf(f, "X-RSSyl-URL: %s\n", tmpurl);
 464
 465         if( ritem->last_update > 0) {
 466                 fprintf(f, "X-RSSyl-Last-Seen: %lld\n", (long long)ritem->last_update);
 467         }
 468
 469         /* Message-ID */
 470         if( (tmpid = feed_item_get_id(feed_item)) == NULL )
 471                 tmpid = feed_item_get_url(feed_item);
 472         if( tmpid != NULL )
 473                 fprintf(f, "Message-ID: <%s>\n", tmpid);
 474
 475         /* X-RSSyl-Comments */
 476         if( (text = feed_item_get_comments_url(feed_item)) != NULL )
 477                 fprintf(f, "X-RSSyl-Comments: %s\n", text);
 478
 479         /* References */
 480         if( (text = feed_item_get_parent_id(feed_item)) != NULL )
 481                 fprintf(f, "References: <%s>\n", text);
 482
 483         /* Content-Type */
 484         text = feed_item_get_text(feed_item);
 485         if( text && g_utf8_validate(text, -1, NULL) ) {
 486                 fprintf(f, "Content-Type: text/html; charset=UTF-8\n\n");
 487                 meta_charset = g_strdup("<meta http-equiv=\"Content-Type\" "
 488                                 "content=\"text/html; charset=UTF-8\">");
 489         } else {
 490                 fprintf(f, "Content-Type: text/html\n\n");
 491         }
 492
 493         /* construct base href */
 494         if( feed_item_get_url(feed_item) != NULL )
 495                 baseurl = g_strdup_printf("<base href=\"%s\">\n",
 496                         feed_item_get_url(feed_item) );
 497
 498         if( ritem->write_heading )
 499                 heading = g_strdup_printf("<h2>%s</h2>\n<br><br>\n",
 500                                 feed_item_get_title(feed_item));
 501
 502         /* Message body */
 503         fprintf(f, "<html><head>"
 504                         "%s\n"
 505                         "%s"
 506                         "</head>\n<body>\n"
 507                         "%s\n"
 508                         "URL: <a href=\"%s\">%s</a>\n\n<br><br>\n"
 509                         RSSYL_TEXT_START"\n"
 510                         "%s%s"
 511                         RSSYL_TEXT_END"\n\n",
 512                         (meta_charset ? meta_charset : ""),
 513                         (baseurl ? baseurl : ""),
 514                         (heading ? heading : ""),
 515                         (tmpurl ? tmpurl : ""),
 516                         (tmpurl ? tmpurl : "n/a"),
 517                         (text ? text : ""), (text ? "\n" : "") );
 518
 519         g_free(meta_charset);
 520         g_free(baseurl);
 521         g_free(heading);
 522
 523         if( (enc = feed_item_get_enclosure(feed_item)) != NULL )
 524                 fprintf(f, "<p><a href=\"%s\">Attached media file</a> [%s] (%ld bytes)</p>\n",
 525                                 feed_item_enclosure_get_url(enc),
 526                                 feed_item_enclosure_get_type(enc),
 527                                 feed_item_enclosure_get_size(enc) );
 528
 529         fprintf(f, "</body></html>\n");
 530         fclose(f);
 531
 532         g_return_if_fail(template != NULL);
 533
 534         flags = g_new(MsgFlags, 1);
 535         flags->perm_flags = MSG_NEW | MSG_UNREAD;
 536         flags->tmp_flags = 0;
 537
 538         d = folder_item_add_msg(&ritem->item, template, flags, TRUE);
 539         g_free(template);
 540         g_free(flags);
 541
 542         ctx = g_new0(RFeedCtx, 1);
 543         ctx->path = (gpointer)g_strdup_printf("%s%c%d", dirname,
 544                         G_DIR_SEPARATOR, d);
 545         ctx->last_seen = ritem->last_update;
 546         ((FeedItem *)ritem->items->data)->data = (gpointer)ctx;
 547
 548         g_free(dirname);
 549
 550         /* Unset unread+new if the changed item wasn't set unread and user
 551          * doesn't want to see it unread because of the change. */
 552         if (dif != EXISTS_NEW) {
 553                 if (!(oldperm_flags & MSG_UNREAD) && (ritem->silent_update == 2
 554                                 || (ritem->silent_update == 1 && dif == EXISTS_CHANGED_TEXTONLY)))
 555                         procmsg_msginfo_unset_flags(
 556                                         folder_item_get_msginfo((FolderItem *)ritem, d), MSG_NEW | MSG_UNREAD, 0);
 557         }
 558
 559         debug_print("RSSyl: folder_item_add_msg(): %d\n", d);
 560 }