Debug bug #2257
[claws.git] / src / imap.c
index ddeceba5b19b4436e8f44f566e1ee0bd4440dff1..6ebcd3376df07919c0ad1ba507b0c9ffed92b1e4 100644 (file)
@@ -95,6 +95,8 @@ struct _IMAPFolder
        gchar last_seen_separator;
        guint refcnt;
        guint max_set_size;
+       const gchar *search_charset;
+       gboolean search_charset_supported;
 };
 
 struct _IMAPSession
@@ -122,6 +124,7 @@ struct _IMAPSession
        gboolean busy;
        gboolean cancelled;
        gboolean sens_update_block;
+       gboolean do_destroy;
 };
 
 struct _IMAPNameSpace
@@ -223,6 +226,14 @@ static gint        imap_copy_msgs          (Folder         *folder,
                                         MsgInfoList    *msglist, 
                                         GHashTable     *relation);
 
+static gint    search_msgs             (Folder                 *folder,
+                                        FolderItem             *container,
+                                        MsgNumberList          **msgs,
+                                        gboolean               *on_server,
+                                        MatcherList            *predicate,
+                                        SearchProgressNotify   progress_cb,
+                                        gpointer               progress_data);
+
 static gint    imap_remove_msg         (Folder         *folder, 
                                         FolderItem     *item, 
                                         gint            uid);
@@ -422,7 +433,7 @@ static GSList * imap_list_from_lep(IMAPFolder * folder,
                                   clist * list, const gchar * real_path, gboolean all);
 static GSList * imap_get_lep_set_from_numlist(IMAPFolder *folder, MsgNumberList *numlist);
 static GSList * imap_get_lep_set_from_msglist(IMAPFolder *folder, MsgInfoList *msglist);
-static GSList * imap_uid_list_from_lep(clist * list);
+static GSList * imap_uid_list_from_lep(clist * list, gint* length);
 static GSList * imap_uid_list_from_lep_tab(carray * list);
 static void imap_flags_hash_from_lep_uid_flags_tab(carray * list,
                                                   GHashTable * hash,
@@ -445,7 +456,7 @@ FolderClass *imap_get_class(void)
                imap_class.type = F_IMAP;
                imap_class.idstr = "imap";
                imap_class.uistr = "IMAP4";
-               imap_class.supports_server_search = FALSE;
+               imap_class.supports_server_search = TRUE;
 
                /* Folder functions */
                imap_class.new_folder = imap_folder_new;
@@ -477,7 +488,7 @@ FolderClass *imap_get_class(void)
                imap_class.add_msgs = imap_add_msgs;
                imap_class.copy_msg = imap_copy_msg;
                imap_class.copy_msgs = imap_copy_msgs;
-               imap_class.search_msgs = folder_item_search_msgs_local;
+               imap_class.search_msgs = search_msgs;
                imap_class.remove_msg = imap_remove_msg;
                imap_class.remove_msgs = imap_remove_msgs;
                imap_class.expunge = imap_expunge;
@@ -534,6 +545,24 @@ static void unlock_session(IMAPSession *session)
        }
 }
 
+static gboolean imap_ping(gpointer data)
+{
+       Session *session = (Session *)data;
+       IMAPSession *imap_session = IMAP_SESSION(session);
+       int r;
+
+       if (session->state != SESSION_READY)
+               return FALSE;
+       if (imap_session->busy || !imap_session->authenticated)
+               return TRUE;
+       
+       lock_session(imap_session);
+       r = imap_cmd_noop(imap_session);
+       unlock_session(imap_session);
+
+       return r == MAILIMAP_NO_ERROR;
+}
+
 static void imap_disc_session_destroy(IMAPSession *session)
 {
        RemoteFolder *rfolder = NULL;
@@ -550,6 +579,14 @@ static void imap_disc_session_destroy(IMAPSession *session)
        SESSION(session)->sock = NULL;
 }
 
+static void imap_safe_destroy(IMAPSession *session)
+{
+       if (!session->busy)
+               session_destroy(SESSION(session));
+       else
+               session->do_destroy = TRUE;
+}
+
 static gboolean is_fatal(int libetpan_errcode)
 {
        switch(libetpan_errcode) {
@@ -732,6 +769,8 @@ static void imap_folder_destroy(Folder *folder)
        while (imap_folder_get_refcnt(folder) > 0)
                gtk_main_iteration();
        
+       g_free(IMAP_FOLDER(folder)->search_charset);
+
        folder_remote_folder_destroy(REMOTE_FOLDER(folder));
        imap_done(folder);
 }
@@ -741,6 +780,8 @@ static void imap_folder_init(Folder *folder, const gchar *name,
 {
        folder_remote_folder_init((Folder *)folder, name, path);
        IMAP_FOLDER(folder)->max_set_size = IMAP_SET_MAX_COUNT;
+       IMAP_FOLDER(folder)->search_charset_supported = TRUE;
+       IMAP_FOLDER(folder)->search_charset = conv_get_locale_charset_str_no_utf8();
 }
 
 static FolderItem *imap_folder_item_new(Folder *folder)
@@ -922,7 +963,7 @@ static IMAPSession *imap_reconnect_if_possible(Folder *folder, IMAPSession *sess
                log_warning(LOG_PROTOCOL, _("Connecting to %s failed"),
                            folder->account->recv_server);
                SESSION(session)->sock = NULL;
-               session_destroy(SESSION(session));
+               imap_safe_destroy(session);
                session = NULL;
        } else {
                rfolder->session = NULL;
@@ -934,7 +975,7 @@ static IMAPSession *imap_reconnect_if_possible(Folder *folder, IMAPSession *sess
                            folder->account->recv_server);
                SESSION(session)->state = SESSION_DISCONNECTED;
                SESSION(session)->sock = NULL;
-               session_destroy(SESSION(session));
+               imap_safe_destroy(session);
                /* Clear folders session to make imap_session_get create
                   a new session, because of rfolder->session == NULL
                   it will not try to reconnect again and so avoid an
@@ -964,11 +1005,21 @@ static IMAPSession *imap_session_get(Folder *folder)
                return NULL;
        }
 
+       /* check for deferred destroy */
+       if (rfolder->session != NULL) {
+               session = IMAP_SESSION(rfolder->session);
+               if (!session->busy && session->do_destroy) {
+                       rfolder->session = NULL;
+                       imap_safe_destroy(session);
+                       session = NULL;
+               }
+       }
+
        /* Make sure we have a session */
        if (rfolder->session != NULL && rfolder->session->state != SESSION_DISCONNECTED) {
                session = IMAP_SESSION(rfolder->session);
        } else if (rfolder->session != NULL && rfolder->session->state == SESSION_DISCONNECTED) {
-               session_destroy(SESSION(rfolder->session));
+               imap_safe_destroy(IMAP_SESSION(rfolder->session));
                rfolder->session = NULL;
                goto new_conn;
        } else if (rfolder->connecting) {
@@ -998,7 +1049,7 @@ new_conn:
                        imap_threaded_disconnect(session->folder);
                        SESSION(session)->state = SESSION_DISCONNECTED;
                        SESSION(session)->sock = NULL;
-                       session_destroy(SESSION(session));
+                       imap_safe_destroy(session);
                }
                rfolder->last_failure = time(NULL);
                rfolder->connecting = FALSE;
@@ -1157,7 +1208,7 @@ static IMAPSession *imap_session_new(Folder * folder,
                        log_warning(LOG_PROTOCOL, _("Can't start TLS session.\n"));
                        if (!is_fatal(ok)) {
                                SESSION(session)->sock = NULL;
-                               session_destroy(SESSION(session));
+                               imap_safe_destroy(session);
                        }
                        return NULL;
                }
@@ -1171,6 +1222,8 @@ static IMAPSession *imap_session_new(Folder * folder,
        log_message(LOG_PROTOCOL, "IMAP connection is %s-authenticated\n",
                    (session->authenticated) ? "pre" : "un");
        
+       session_register_ping(SESSION(session), imap_ping);
+
        return session;
 }
 
@@ -1181,7 +1234,14 @@ static gint imap_session_authenticate(IMAPSession *session,
        gboolean failed = FALSE;
        gint ok = MAILIMAP_NO_ERROR;
        g_return_val_if_fail(account->userid != NULL, MAILIMAP_ERROR_BAD_STATE);
-       acc_pass = account->passwd;
+
+       if (password_get(account->userid, account->recv_server, "imap",
+                        SESSION(session)->port, &pass)) {
+               Xstrdup_a(acc_pass, pass, {g_free(pass); return MAILIMAP_NO_ERROR;});
+               g_free(pass);
+       } else {
+               acc_pass = account->passwd;
+       }
 try_again:
        pass = acc_pass;
        if (!pass && account->imap_auth_type != IMAP_AUTH_ANON && account->imap_auth_type != IMAP_AUTH_GSSAPI) {
@@ -1371,8 +1431,7 @@ static void imap_commit_tags(FolderItem *item, MsgInfo *msginfo, GSList *tags_se
                if (list_set) {
                        ok = imap_set_message_flags(session, 
                                IMAP_FOLDER_ITEM(item), &numlist, 0, list_set, TRUE);
-                       slist_free_strings(list_set);
-                       g_slist_free(list_set);
+                       slist_free_strings_full(list_set);
                        if (ok != MAILIMAP_NO_ERROR) {
                                return;
                        }
@@ -1387,8 +1446,7 @@ static void imap_commit_tags(FolderItem *item, MsgInfo *msginfo, GSList *tags_se
                if (list_unset) {
                        ok = imap_set_message_flags(session, 
                                IMAP_FOLDER_ITEM(item), &numlist, 0, list_unset, FALSE);
-                       slist_free_strings(list_unset);
-                       g_slist_free(list_unset);
+                       slist_free_strings_full(list_unset);
                        if (ok != MAILIMAP_NO_ERROR) {
                                return;
                        }
@@ -1429,13 +1487,14 @@ static gchar *imap_fetch_msg_full(Folder *folder, FolderItem *item, gint uid,
                if (!cached || !MSG_IS_FULLY_CACHED(cached->flags)) {
                        have_size = get_file_size_with_crs(filename);
                        if (cached && (cached->size <= have_size || !body)) {
-                               procmsg_msginfo_free(cached);
                                ok = file_strip_crs(filename);
                                if (ok == 0 && cached && cached->size <= have_size) {
                                        /* we have it all and stripped */
-                                       debug_print("...fully cached in fact; setting flag.\n");
+                                       debug_print("...fully cached in fact (%d/%d); setting flag.\n",
+                                                       have_size, cached->size);
                                        procmsg_msginfo_set_flags(cached, MSG_FULLY_CACHED, 0);
                                }
+                               procmsg_msginfo_free(cached);
                                return filename;
                        } else if (!cached && time(NULL) - get_file_mtime(filename) < 60) {
                                debug_print("message not cached and file recent, considering file complete\n");
@@ -1523,8 +1582,10 @@ static gboolean imap_is_msg_fully_cached(Folder *folder, FolderItem *item, gint
                return TRUE;
        }
        path = folder_item_get_path(item);
-       if (!is_dir_exist(path))
+       if (!is_dir_exist(path)) {
+               g_free(path);
                return FALSE;
+       }
 
        filename = g_strconcat(path, G_DIR_SEPARATOR_S, itos(uid), NULL);
        g_free(path);
@@ -1538,6 +1599,7 @@ static gboolean imap_is_msg_fully_cached(Folder *folder, FolderItem *item, gint
                size = get_file_size_with_crs(filename);
                g_free(filename);
        }
+       debug_print("msg %d cached, has size %d, full should be %d.\n", uid, size, cached->size);
        if (cached && size >= cached->size) {
                cached->total_size = cached->size;
                procmsg_msginfo_set_flags(cached, MSG_FULLY_CACHED, 0);
@@ -1951,6 +2013,400 @@ static gint imap_copy_msgs(Folder *folder, FolderItem *dest,
        return ret;
 }
 
+static gboolean imap_matcher_type_is_local(gint matchertype)
+{
+       switch (matchertype) {
+       case MATCHCRITERIA_FROM:
+       case MATCHCRITERIA_TO:
+       case MATCHCRITERIA_CC:
+       case MATCHCRITERIA_TO_OR_CC:
+       case MATCHCRITERIA_SUBJECT:
+       case MATCHCRITERIA_REFERENCES:
+       case MATCHCRITERIA_INREPLYTO:
+       case MATCHCRITERIA_AGE_GREATER:
+       case MATCHCRITERIA_AGE_LOWER:
+       case MATCHCRITERIA_FORWARDED:
+       case MATCHCRITERIA_SPAM:
+       case MATCHCRITERIA_UNREAD:
+       case MATCHCRITERIA_NEW:
+       case MATCHCRITERIA_MARKED:
+       case MATCHCRITERIA_REPLIED:
+       case MATCHCRITERIA_DELETED:
+       case MATCHCRITERIA_SIZE_GREATER:
+       case MATCHCRITERIA_SIZE_SMALLER:
+       case MATCHCRITERIA_SIZE_EQUAL:
+               return TRUE;
+       }
+       return FALSE;
+}
+
+static IMAPSearchKey* search_make_key(MatcherProp* match, gboolean* is_all)
+{
+       if (match->matchtype == MATCHTYPE_MATCHCASE || match->matchtype == MATCHTYPE_MATCH) {
+               IMAPSearchKey* result = NULL;
+               gboolean invert = FALSE;
+               gint matchertype = match->criteria;
+
+               if (is_all) {
+                       *is_all = FALSE;
+               }
+
+               switch (matchertype) {
+               case MATCHCRITERIA_NOT_NEW: invert = TRUE; matchertype = MATCHCRITERIA_NEW; break;
+               case MATCHCRITERIA_NOT_MARKED: invert = TRUE; matchertype = MATCHCRITERIA_MARKED; break;
+               case MATCHCRITERIA_NOT_FORWARDED: invert = TRUE; matchertype = MATCHCRITERIA_FORWARDED; break;
+               case MATCHCRITERIA_NOT_SPAM: invert = TRUE; matchertype = MATCHCRITERIA_SPAM; break;
+               case MATCHCRITERIA_NOT_SUBJECT: invert = TRUE; matchertype = MATCHCRITERIA_SUBJECT; break;
+               case MATCHCRITERIA_NOT_FROM: invert = TRUE; matchertype = MATCHCRITERIA_FROM; break;
+               case MATCHCRITERIA_NOT_TO: invert = TRUE; matchertype = MATCHCRITERIA_TO; break;
+               case MATCHCRITERIA_NOT_CC: invert = TRUE; matchertype = MATCHCRITERIA_CC; break;
+               case MATCHCRITERIA_NOT_REFERENCES: invert = TRUE; matchertype = MATCHCRITERIA_REFERENCES; break;
+               case MATCHCRITERIA_NOT_HEADER: invert = TRUE; matchertype = MATCHCRITERIA_HEADER; break;
+               case MATCHCRITERIA_NOT_TAG: invert = TRUE; matchertype = MATCHCRITERIA_TAG; break;
+               case MATCHCRITERIA_NOT_HEADERS_PART: invert = TRUE; matchertype = MATCHCRITERIA_HEADERS_PART; break;
+               case MATCHCRITERIA_NOT_MESSAGE: invert = TRUE; matchertype = MATCHCRITERIA_MESSAGE; break;
+               case MATCHCRITERIA_NOT_BODY_PART: invert = TRUE; matchertype = MATCHCRITERIA_BODY_PART; break;
+               case MATCHCRITERIA_NOT_TO_AND_NOT_CC: invert = TRUE; matchertype = MATCHCRITERIA_TO_OR_CC; break;
+               case MATCHCRITERIA_NOT_INREPLYTO: invert = TRUE; matchertype = MATCHCRITERIA_INREPLYTO; break;
+               }
+
+               /* 
+                * this aborts conversion even for predicates understood by the following code.
+                * while that might seem wasteful, claws local search for information listed below
+                * has proven faster than IMAP search plus network roundtrips. once this changes,
+                * consider removing these exceptions.
+                */
+               if (imap_matcher_type_is_local(matchertype))
+                       return NULL;
+
+               /* the Message-ID header is also cached */
+               if (matchertype == MATCHCRITERIA_HEADER && g_strcmp0("Message-ID", match->header) == 0) {
+                       return NULL;
+               }
+
+               switch (matchertype) {
+               case MATCHCRITERIA_FORWARDED:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_TAG, NULL, RTAG_FORWARDED, 0);
+                       break;
+
+               case MATCHCRITERIA_SPAM:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_TAG, NULL, RTAG_JUNK, 0);
+                       break;
+
+               case MATCHCRITERIA_INREPLYTO:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_HEADER, "In-Reply-To", match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_REFERENCES:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_HEADER, "References", match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_TO_OR_CC:
+                       result = imap_search_or(
+                                       imap_search_new(IMAP_SEARCH_CRITERIA_TO, NULL, match->expr, 0),
+                                       imap_search_new(IMAP_SEARCH_CRITERIA_CC, NULL, match->expr, 0)
+                                       );
+                       break;
+
+               case MATCHCRITERIA_HEADERS_PART:
+                       result = imap_search_and(
+                                       imap_search_not(imap_search_new(IMAP_SEARCH_CRITERIA_BODY, NULL, match->expr, 0)),
+                                       imap_search_new(IMAP_SEARCH_CRITERIA_MESSAGE, NULL, match->expr, 0)
+                                       );
+                       break;
+
+               case MATCHCRITERIA_SIZE_EQUAL:
+                       result = imap_search_and(
+                                       imap_search_not(imap_search_new(IMAP_SEARCH_CRITERIA_SIZE_SMALLER, NULL, NULL, match->value)),
+                                       imap_search_not(imap_search_new(IMAP_SEARCH_CRITERIA_SIZE_GREATER, NULL, NULL, match->value))
+                                       );
+                       break;
+
+               case MATCHCRITERIA_NOT_UNREAD:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_READ, NULL, NULL, 0);
+                       break;
+
+               case MATCHCRITERIA_UNREAD:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_UNREAD, NULL, NULL, 0);
+                       break;
+
+               case MATCHCRITERIA_NEW:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_NEW, NULL, NULL, 0);
+                       break;
+
+               case MATCHCRITERIA_MARKED:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_MARKED, NULL, NULL, 0);
+                       break;
+
+               case MATCHCRITERIA_DELETED:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_DELETED, NULL, NULL, 0);
+                       break;
+
+               case MATCHCRITERIA_REPLIED:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_REPLIED, NULL, NULL, 0);
+                       break;
+
+               case MATCHCRITERIA_TAG:
+                       {
+                               gchar *tmp = imap_utf8_to_modified_utf7(match->expr, TRUE);
+                               result = imap_search_new(IMAP_SEARCH_CRITERIA_TAG, NULL, tmp, 0);
+                               g_free(tmp);
+                       }
+                       break;
+
+               case MATCHCRITERIA_SUBJECT:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_SUBJECT, NULL, match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_FROM:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_FROM, NULL, match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_TO:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_TO, NULL, match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_CC:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_CC, NULL, match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_AGE_GREATER:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_AGE_GREATER, NULL, NULL, match->value);
+                       break;
+
+               case MATCHCRITERIA_AGE_LOWER:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_AGE_LOWER, NULL, NULL, match->value);
+                       break;
+
+               case MATCHCRITERIA_BODY_PART:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_BODY, NULL, match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_MESSAGE:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_MESSAGE, NULL, match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_HEADER:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_HEADER, match->header, match->expr, 0);
+                       break;
+
+               case MATCHCRITERIA_SIZE_GREATER:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_SIZE_GREATER, NULL, NULL, match->value);
+                       break;
+
+               case MATCHCRITERIA_SIZE_SMALLER:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_SIZE_SMALLER, NULL, NULL, match->value);
+                       break;
+
+               default:
+                       result = imap_search_new(IMAP_SEARCH_CRITERIA_ALL, NULL, NULL, 0);
+                       if (is_all) {
+                               *is_all = TRUE;
+                       }
+                       break;
+               }
+
+               if (invert) {
+                       result = imap_search_not(result);
+                       if (is_all && *is_all) {
+                               *is_all = FALSE;
+                       }
+               }
+
+               return result;
+       }
+
+       return NULL;
+}
+
+static void imap_change_search_charset(IMAPFolder *folder)
+{
+       /* If server supports charset in searches, but the last used one failed,
+        * changed to the next preferred charset. If none are still available,
+        * disable charset searches.
+        * Charsets are tried in the following order: 
+        * UTF-8, locale's charset, UTF-7.
+        */
+
+       if (folder->search_charset_supported) {
+               if (folder->search_charset && !strcmp(folder->search_charset, conv_get_locale_charset_str_no_utf8()))
+                       folder->search_charset = "UTF-8";
+               else if (folder->search_charset && !strcmp(folder->search_charset, "UTF-8"))
+                       folder->search_charset = "UTF-7";
+               else {
+                       folder->search_charset = NULL;
+                       folder->search_charset_supported = FALSE;
+               }
+       }
+}
+
+static MatcherProp *imap_matcher_prop_set_charset(IMAPFolder *folder,
+                                                 MatcherProp *utf8_prop,
+                                                 gchar **charset)
+{
+       /* If the match is going to be done locally, or the criteria is on
+        * tag (special-cased to modified-UTF-7), or the expression searched
+        * is ASCII, don't bother converting.
+        */
+       if (imap_matcher_type_is_local(utf8_prop->criteria)
+        || utf8_prop->criteria == MATCHCRITERIA_TAG
+        || utf8_prop->criteria == MATCHCRITERIA_NOT_TAG
+        || utf8_prop->expr == NULL
+        || is_ascii_str(utf8_prop->expr))
+               return matcherprop_new(utf8_prop->criteria,
+                              utf8_prop->header,
+                              utf8_prop->matchtype,
+                              utf8_prop->expr,
+                              utf8_prop->value);
+       else {
+               gchar *conv_expr = NULL;
+
+               /* If the search is server-side and the server doesn't support
+                * searching with the charsets we handle, bail out.
+                */
+               if (folder->search_charset_supported == FALSE)
+                       return NULL;
+
+               /* Else, convert. */
+               if (*charset == NULL)
+                       *charset = g_strdup(folder->search_charset);
+
+               conv_expr = conv_codeset_strdup(utf8_prop->expr, CS_UTF_8, *charset);
+
+               if (conv_expr == NULL)
+                       conv_expr = g_strdup(utf8_prop->expr);
+
+               return matcherprop_new(utf8_prop->criteria,
+                              utf8_prop->header,
+                              utf8_prop->matchtype,
+                              conv_expr,
+                              utf8_prop->value);
+       }
+}
+
+static gint    search_msgs             (Folder                 *folder,
+                                        FolderItem             *container,
+                                        MsgNumberList          **msgs,
+                                        gboolean               *on_server,
+                                        MatcherList            *predicate,
+                                        SearchProgressNotify   progress_cb,
+                                        gpointer               progress_data)
+{
+       IMAPSearchKey* key = NULL;
+       GSList* cur;
+       int result = -1;
+       clist* uidlist = NULL;
+       gboolean server_filtering_useless = FALSE;
+        IMAPSession *session;
+       gchar *charset_to_use = NULL;
+
+       if (on_server == NULL || !*on_server) {
+               return folder_item_search_msgs_local(folder, container, msgs, on_server,
+                               predicate, progress_cb, progress_data);
+       }
+
+       for (cur = predicate->matchers; cur != NULL; cur = cur->next) {
+               IMAPSearchKey* matcherPart = NULL;
+               MatcherProp* prop = (MatcherProp*) cur->data;
+               gboolean is_all;
+               MatcherProp *imap_prop = imap_matcher_prop_set_charset(IMAP_FOLDER(folder), prop, &charset_to_use);
+
+               if (imap_prop == NULL) {
+                       /* Couldn't convert matcherprop to IMAP - probably not ascii
+                        * and server doesn't support the charsets we do. */
+                        return -1;
+               }
+
+               matcherPart = search_make_key(imap_prop, &is_all);
+
+               matcherprop_free(imap_prop);
+
+               if (on_server) {
+                       *on_server &= matcherPart != NULL && prop->matchtype == MATCHTYPE_MATCHCASE;
+               }
+
+               if (matcherPart) {
+                       if (key == NULL) {
+                               key = matcherPart;
+                               server_filtering_useless = is_all;
+                       } else if (predicate->bool_and) {
+                               key = imap_search_and(key, matcherPart);
+                               server_filtering_useless &= is_all;
+                       } else {
+                               key = imap_search_or(key, matcherPart);
+                               server_filtering_useless |= is_all;
+                       }
+               }
+       }
+
+       if (server_filtering_useless) {
+               imap_search_free(key);
+               key = NULL;
+       }
+
+       if (key == NULL && progress_cb != NULL) {
+               GSList* cur;
+               GSList* list;
+               int count = 0;
+
+               progress_cb(progress_data, TRUE, 0, 0, container->total_msgs);
+               progress_cb(progress_data, TRUE, container->total_msgs, 0, container->total_msgs);
+
+               list = folder_item_get_msg_list(container);
+               for (cur = list; cur != NULL; cur = cur->next) {
+                       *msgs = g_slist_prepend(*msgs, GUINT_TO_POINTER(((MsgInfo*) cur->data)->msgnum));
+                       count++;
+               }
+               procmsg_msg_list_free(list);
+
+               *msgs = g_slist_reverse(*msgs);
+
+               return count;
+       }
+
+       session = imap_session_get(folder);
+        if (!session) {
+                return -1;
+        }
+       result = imap_select(session, IMAP_FOLDER(folder), FOLDER_ITEM(container),
+                        NULL, NULL, NULL, NULL, NULL, TRUE);
+       if (result != MAILIMAP_NO_ERROR)
+               return -1;
+
+       if (progress_cb)
+               progress_cb(progress_data, TRUE, 0, 0, container->total_msgs);
+       result = imap_threaded_search(folder, IMAP_SEARCH_TYPE_KEYED, key, charset_to_use, NULL, &uidlist);
+       if (progress_cb)
+               progress_cb(progress_data, TRUE, container->total_msgs, 0, container->total_msgs);
+
+       if (result == MAILIMAP_NO_ERROR) {
+               gint result = 0;
+
+               *msgs = imap_uid_list_from_lep(uidlist, &result);
+
+               mailimap_search_result_free(uidlist);
+
+               if (charset_to_use != NULL)
+                       g_free(charset_to_use);
+
+               return result;
+       } else if (charset_to_use != NULL) {
+               /* If search failed and was on an 8-bit string, try the next
+                * available charset to search if there still are some.
+                */
+               g_free(charset_to_use);
+               
+               imap_change_search_charset(IMAP_FOLDER(folder));
+               if (IMAP_FOLDER(folder)->search_charset_supported)
+                       return search_msgs(folder, container, msgs, on_server, predicate,
+                                  progress_cb, progress_data);
+               else
+                       return -1;
+       } else {
+               return -1;
+       }
+}
+
 
 static gint imap_do_remove_msgs(Folder *folder, FolderItem *dest, 
                                MsgInfoList *msglist, GHashTable *relation)
@@ -3010,8 +3466,7 @@ static void *imap_get_uncached_messages_thread(void *data)
                        tags = carray_get(env_list, i+1);
                        msginfo = imap_envelope_from_lep(info, item);
                        if (msginfo == NULL) {
-                               slist_free_strings(tags);
-                               g_slist_free(tags);
+                               slist_free_strings_full(tags);
                                continue;
                        }
                        g_slist_free(msginfo->tags);
@@ -3034,8 +3489,7 @@ static void *imap_get_uncached_messages_thread(void *data)
                        }
                        if (msginfo->tags)
                                msginfo->tags = g_slist_reverse(msginfo->tags);
-                       slist_free_strings(tags);
-                       g_slist_free(tags);
+                       slist_free_strings_full(tags);
                        msginfo->folder = item;
                        if (!newlist)
                                llast = newlist = g_slist_append(newlist, msginfo);
@@ -3054,12 +3508,7 @@ static void *imap_get_uncached_messages_thread(void *data)
                main_window_reflect_tags_changes(mainwindow_get_mainwindow());
        }
 
-       for (cur = seq_list; cur != NULL; cur = g_slist_next(cur)) {
-               struct mailimap_set * imapset;
-               
-               imapset = cur->data;
-               mailimap_set_free(imapset);
-       }
+       imap_lep_set_free(seq_list);
        
        session_set_access_time(SESSION(session));
        stuff->done = TRUE;
@@ -3519,8 +3968,7 @@ static gint imap_status(IMAPSession *session, IMAPFolder *folder,
 
 static void imap_free_capabilities(IMAPSession *session)
 {
-       slist_free_strings(session->capability);
-       g_slist_free(session->capability);
+       slist_free_strings_full(session->capability);
        session->capability = NULL;
 }
 
@@ -3595,8 +4043,6 @@ static gint imap_cmd_noop(IMAPSession *session)
                return r;
        }
 
-       session->folder_content_changed = FALSE;
-
        if ((exists && exists != session->exists)
         || (recent && recent != session->recent)
         || (expunge && expunge != session->expunge)
@@ -3980,13 +4426,13 @@ static gint get_list_of_uids(IMAPSession *session, Folder *folder, IMAPFolderIte
        uidlist = NULL;
        
        if (folder->account && folder->account->low_bandwidth) {
-               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_SIMPLE, NULL,
-                                &lep_uidlist);
+               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_SIMPLE,
+                               NULL, NULL, NULL, &lep_uidlist);
        }
        
        if (r == MAILIMAP_NO_ERROR) {
                GSList * fetchuid_list =
-                       imap_uid_list_from_lep(lep_uidlist);
+                       imap_uid_list_from_lep(lep_uidlist, NULL);
                mailimap_search_result_free(lep_uidlist);
                
                uidlist = g_slist_concat(fetchuid_list, uidlist);
@@ -4071,12 +4517,12 @@ gint imap_get_num_list(Folder *folder, FolderItem *_item, GSList **msgnum_list,
 
        lock_session(session); /* unlocked by get_list_of_uids */
        if (FOLDER_ITEM(item)->path) 
-               statusbar_print_all(_("Scanning folder %s%c%s ..."),
+               statusbar_print_all(_("Scanning folder %s%c%s..."),
                                      FOLDER_ITEM(item)->folder->name, 
                                      G_DIR_SEPARATOR,
                                      FOLDER_ITEM(item)->path);
        else
-               statusbar_print_all(_("Scanning folder %s ..."),
+               statusbar_print_all(_("Scanning folder %s..."),
                                      FOLDER_ITEM(item)->folder->name);
 
        if (item->should_trash_cache) {
@@ -4615,18 +5061,18 @@ static /*gint*/ void *imap_get_flags_thread(void *data)
 
                        imapset = cur->data;
                        if (reverse_seen) {
-                               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_SEEN,
-                                                        full_search ? NULL:imapset, &lep_uidlist);
+                               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_SEEN, NULL,
+                                                        NULL, full_search ? NULL:imapset, &lep_uidlist);
                        }
                        else {
                                r = imap_threaded_search(folder,
-                                                        IMAP_SEARCH_TYPE_UNSEEN,
-                                                        full_search ? NULL:imapset, &lep_uidlist);
+                                                        IMAP_SEARCH_TYPE_UNSEEN, NULL,
+                                                        NULL, full_search ? NULL:imapset, &lep_uidlist);
                        }
                        if (r == MAILIMAP_NO_ERROR) {
                                GSList * uidlist;
 
-                               uidlist = imap_uid_list_from_lep(lep_uidlist);
+                               uidlist = imap_uid_list_from_lep(lep_uidlist, NULL);
                                mailimap_search_result_free(lep_uidlist);
 
                                unseen = g_slist_concat(unseen, uidlist);
@@ -4635,12 +5081,12 @@ static /*gint*/ void *imap_get_flags_thread(void *data)
                                goto bail;
                        }
 
-                       r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_FLAGGED,
-                                                full_search ? NULL:imapset, &lep_uidlist);
+                       r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_FLAGGED, NULL,
+                                                NULL, full_search ? NULL:imapset, &lep_uidlist);
                        if (r == MAILIMAP_NO_ERROR) {
                                GSList * uidlist;
 
-                               uidlist = imap_uid_list_from_lep(lep_uidlist);
+                               uidlist = imap_uid_list_from_lep(lep_uidlist, NULL);
                                mailimap_search_result_free(lep_uidlist);
 
                                flagged = g_slist_concat(flagged, uidlist);
@@ -4650,12 +5096,12 @@ static /*gint*/ void *imap_get_flags_thread(void *data)
                        }
 
                        if (fitem->opened || fitem->processing_pending || fitem == folder->inbox) {
-                               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_ANSWERED,
-                                                        full_search ? NULL:imapset, &lep_uidlist);
+                               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_ANSWERED, NULL,
+                                                        NULL, full_search ? NULL:imapset, &lep_uidlist);
                                if (r == MAILIMAP_NO_ERROR) {
                                        GSList * uidlist;
 
-                                       uidlist = imap_uid_list_from_lep(lep_uidlist);
+                                       uidlist = imap_uid_list_from_lep(lep_uidlist, NULL);
                                        mailimap_search_result_free(lep_uidlist);
 
                                        answered = g_slist_concat(answered, uidlist);
@@ -4665,12 +5111,12 @@ static /*gint*/ void *imap_get_flags_thread(void *data)
                                }
 
                                if (flag_ok(IMAP_FOLDER_ITEM(fitem), IMAP_FLAG_FORWARDED)) {
-                                       r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_FORWARDED,
-                                                                full_search ? NULL:imapset, &lep_uidlist);
+                                       r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_FORWARDED, NULL,
+                                                                NULL, full_search ? NULL:imapset, &lep_uidlist);
                                        if (r == MAILIMAP_NO_ERROR) {
                                                GSList * uidlist;
 
-                                               uidlist = imap_uid_list_from_lep(lep_uidlist);
+                                               uidlist = imap_uid_list_from_lep(lep_uidlist, NULL);
                                                mailimap_search_result_free(lep_uidlist);
 
                                                forwarded = g_slist_concat(forwarded, uidlist);
@@ -4681,12 +5127,12 @@ static /*gint*/ void *imap_get_flags_thread(void *data)
                                }
 
                                if (flag_ok(IMAP_FOLDER_ITEM(fitem), IMAP_FLAG_SPAM)) {
-                                       r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_SPAM,
-                                                                full_search ? NULL:imapset, &lep_uidlist);
+                                       r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_SPAM, NULL,
+                                                                NULL, full_search ? NULL:imapset, &lep_uidlist);
                                        if (r == MAILIMAP_NO_ERROR) {
                                                GSList * uidlist;
 
-                                               uidlist = imap_uid_list_from_lep(lep_uidlist);
+                                               uidlist = imap_uid_list_from_lep(lep_uidlist, NULL);
                                                mailimap_search_result_free(lep_uidlist);
 
                                                spam = g_slist_concat(spam, uidlist);
@@ -4696,12 +5142,12 @@ static /*gint*/ void *imap_get_flags_thread(void *data)
                                        }
                                }
 
-                               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_DELETED,
-                                                        full_search ? NULL:imapset, &lep_uidlist);
+                               r = imap_threaded_search(folder, IMAP_SEARCH_TYPE_DELETED, NULL,
+                                                        NULL, full_search ? NULL:imapset, &lep_uidlist);
                                if (r == MAILIMAP_NO_ERROR) {
                                        GSList * uidlist;
 
-                                       uidlist = imap_uid_list_from_lep(lep_uidlist);
+                                       uidlist = imap_uid_list_from_lep(lep_uidlist, NULL);
                                        mailimap_search_result_free(lep_uidlist);
 
                                        deleted = g_slist_concat(deleted, uidlist);
@@ -4811,8 +5257,7 @@ bail:
                                                                GINT_TO_POINTER(id));
                                        g_free(real_tag);
                                }
-                               slist_free_strings(tags);
-                               g_slist_free(tags);
+                               slist_free_strings_full(tags);
                        }
                }
 
@@ -5226,25 +5671,21 @@ static GSList * imap_get_lep_set_from_numlist(IMAPFolder *folder, MsgNumberList
 static GSList * imap_get_lep_set_from_msglist(IMAPFolder *folder, MsgInfoList *msglist)
 {
        MsgNumberList *numlist = NULL;
-       MsgInfoList *cur;
        GSList *seq_list;
 
-       for (cur = msglist; cur != NULL; cur = g_slist_next(cur)) {
-               MsgInfo *msginfo = (MsgInfo *) cur->data;
+       numlist = procmsg_get_number_list_for_msgs(msglist);
 
-               numlist = g_slist_prepend(numlist, GINT_TO_POINTER(msginfo->msgnum));
-       }
-       numlist = g_slist_reverse(numlist);
        seq_list = imap_get_lep_set_from_numlist(folder, numlist);
        g_slist_free(numlist);
 
        return seq_list;
 }
 
-static GSList * imap_uid_list_from_lep(clist * list)
+static GSList * imap_uid_list_from_lep(clist * list, gint* length)
 {
        clistiter * iter;
        GSList * result;
+       gint len = 0;
        
        result = NULL;
        
@@ -5255,9 +5696,12 @@ static GSList * imap_uid_list_from_lep(clist * list)
 
                        puid = clist_content(iter);
                        result = g_slist_prepend(result, GINT_TO_POINTER(* puid));
+                       len++;
                }
                result = g_slist_reverse(result);
-       }       
+       }
+       if (length)
+               *length = len;
        return result;
 }
 
@@ -5305,7 +5749,7 @@ static MsgInfo *imap_envelope_from_lep(struct imap_fetch_env_info * info,
        guint32 uid = 0;
        goffset size = 0;
        MsgFlags flags = {0, 0};
-       
+
        if (info->headers == NULL)
                return NULL;
 
@@ -5316,11 +5760,11 @@ static MsgInfo *imap_envelope_from_lep(struct imap_fetch_env_info * info,
                MSG_SET_TMP_FLAGS(flags, MSG_DRAFT);
        }
        flags.perm_flags = info->flags;
-       
+
        uid = info->uid;
        size = (goffset) info->size;
        msginfo = procheader_parse_str(info->headers, flags, FALSE, FALSE);
-       
+
        if (msginfo) {
                msginfo->msgnum = uid;
                msginfo->size = size;
@@ -5423,12 +5867,15 @@ void imap_disconnect_all(gboolean have_connectivity)
                if (account->protocol == A_IMAP4) {
                        RemoteFolder *folder = (RemoteFolder *)account->folder;
                        if (folder && folder->session) {
+                               if (imap_is_busy(FOLDER(folder)))
+                                       imap_threaded_cancel(FOLDER(folder));
+
                                IMAPSession *session = (IMAPSession *)folder->session;
                                if (have_connectivity)
                                        imap_threaded_disconnect(FOLDER(folder));
                                SESSION(session)->state = SESSION_DISCONNECTED;
                                SESSION(session)->sock = NULL;
-                               session_destroy(SESSION(session));
+                               imap_safe_destroy(session);
                                folder->session = NULL;
                        }
                }