File bnc-472719-eds-disk-summary-patches.patch of Package evolution-data-server

Overview Repositories Revisions Requests Users Attributes Meta

File bnc-472719-eds-disk-summary-patches.patch of Package evolution-data-server

--- camel/camel-db.c	2009-02-04 09:13:36.000000000 +0530
+++ camel/camel-db.c	2009-02-04 10:19:57.000000000 +0530
@@ -36,9 +36,375 @@
 
 #include "camel-debug.h"
 
+/* how long to wait before invoking sync on the file; in miliseconds */
+#define SYNC_TIMEOUT 5000
+
+static sqlite3_vfs *old_vfs = NULL;
+
+GStaticRecMutex only_once_lock = G_STATIC_REC_MUTEX_INIT;
+GStaticRecMutex sync_queue_lock = G_STATIC_REC_MUTEX_INIT;
+#define LockQueue()   g_static_rec_mutex_lock   (&sync_queue_lock)
+#define UnlockQueue() g_static_rec_mutex_unlock (&sync_queue_lock)
+
+/* 'sync_queue' is using keys sqlite3_file to sync_queue_data structures.
+   Access to this is guarded with LockQueue/UnlockQueue function. */
+static GHashTable *sync_queue = NULL;
+
+typedef struct _sync_queue_data {
+	guint timeout_source; /* id of the source */
+	GThread *running_thread;
+
+	int sync_flags; 
+} sync_queue_data;
+
+struct CamelSqlite3File
+{
+	sqlite3_file parent;
+	sqlite3_file *old_vfs_file; /* pointer to old_vfs' file */
+};
+
+static int
+call_old_file_Sync (sqlite3_file *pFile, int flags)
+{
+	struct CamelSqlite3File *cFile;
+
+	g_return_val_if_fail (old_vfs != NULL, SQLITE_ERROR);
+	g_return_val_if_fail (pFile != NULL, SQLITE_ERROR);
+
+	cFile = (struct CamelSqlite3File *)pFile;
+	g_return_val_if_fail (cFile->old_vfs_file->pMethods != NULL, SQLITE_ERROR);
+	return cFile->old_vfs_file->pMethods->xSync (cFile->old_vfs_file, flags);
+}
+
+static gboolean prepare_to_run_sync_in_thread (gpointer pFile);
+
+static gpointer
+run_sync_in_thread (gpointer pFile)
+{
+	int sync_flags = 0;
+	sync_queue_data *data;
+
+	g_return_val_if_fail (pFile != NULL, NULL);
+	g_return_val_if_fail (sync_queue != NULL, NULL);
+
+	LockQueue ();
+	data = g_hash_table_lookup (sync_queue, pFile);
+	if (data) {
+		/* sync_flags can change while we are running */
+		sync_flags = data->sync_flags;
+		data->sync_flags = 0;
+	}
+	UnlockQueue ();
+
+	/* this should not happen, once we are in a thread, the datas are ours */
+	g_return_val_if_fail (data != NULL, NULL);
+
+	/* do the sync itself, but do not block the sync_queue;
+	   any error here is silently ignored. */
+	call_old_file_Sync (/*sqlite3_file*/pFile, sync_flags);
+
+	LockQueue ();
+	if (data->timeout_source == -1) {
+		/* new sync request arrived meanwhile, indicate thread finished... */
+		data->running_thread = NULL;
+		/* ...and reschedule */
+		data->timeout_source = g_timeout_add (SYNC_TIMEOUT, prepare_to_run_sync_in_thread, pFile);
+	} else {
+		/* remove it from a sync_queue and free memory */
+		g_hash_table_remove (sync_queue, pFile);
+		g_free (data);
+	}
+	UnlockQueue ();
+
+	return NULL;
+}
+
+static gboolean
+prepare_to_run_sync_in_thread (gpointer pFile)
+{
+	sync_queue_data *data;
+
+	g_return_val_if_fail (pFile != NULL, FALSE);
+	g_return_val_if_fail (sync_queue != NULL, FALSE);
+
+	LockQueue ();
+
+	data = g_hash_table_lookup (sync_queue, pFile);
+	/* check if still tracking this file and if didn't get rescheduled */
+	if (data && data->timeout_source == g_source_get_id (g_main_current_source ())) {
+		/* run the thread */
+		data->running_thread = g_thread_create (run_sync_in_thread, pFile, TRUE, NULL);
+		data->timeout_source = 0;
+	}
+
+	UnlockQueue ();
+
+	return FALSE;
+}
+
+/*
+   Adds sync on this file to the queue. Flags are just bit-OR-ed,
+   which will not hopefully hurt. In case the file is waiting for
+   it's sync, we just postpone it once again.
+   In case file is syncing just in call of this, we schedule other
+   sync after that.
+ */
+static void
+queue_sync (sqlite3_file *pFile, int flags)
+{
+	sync_queue_data *data;
+
+	g_return_if_fail (pFile != NULL);
+	g_return_if_fail (flags != 0);
+
+	LockQueue ();
+
+	if (!sync_queue)
+		sync_queue = g_hash_table_new (g_direct_hash, g_direct_equal);
+
+	data = g_hash_table_lookup (sync_queue, pFile);
+	if (data) {
+		/* There is a sync request for this file already. */
+		if (data->running_thread) {
+			/* -1 indicates to reschedule after the actual sync finishes */
+			data->timeout_source = -1;
+			/* start with new flags - thread set it to 0; next time just add others */
+			data->sync_flags = data->sync_flags | flags;
+		} else {
+			data->sync_flags = data->sync_flags | flags;
+
+			/* reschedule */
+			g_source_remove (data->timeout_source);
+			data->timeout_source = g_timeout_add (SYNC_TIMEOUT, prepare_to_run_sync_in_thread, pFile);
+		}
+	} else {
+		data = g_malloc0 (sizeof (sync_queue_data));
+		data->sync_flags = flags;
+		data->running_thread = NULL;
+		data->timeout_source = g_timeout_add (SYNC_TIMEOUT, prepare_to_run_sync_in_thread, pFile);
+
+		g_hash_table_insert (sync_queue, pFile, data);
+	}
+
+	UnlockQueue ();
+}
+
+/*
+   If file is not in a queue, it does nothing, otherwise it removes
+   it from a queue, and calls sync immediately.
+   If file is syncing at the moment, it waits until the previous sync finishes.
+ */
+static void
+dequeue_sync (sqlite3_file *pFile)
+{
+	sync_queue_data *data;
+
+	g_return_if_fail (pFile != NULL);
+
+	LockQueue ();
+
+	if (!sync_queue) {
+		/* closing file which wasn't requested to sync, and none
+		   before it too. */
+		UnlockQueue ();
+		return;
+	}
+
+	data = g_hash_table_lookup (sync_queue, pFile);
+	if (data) {
+		int sync_flags = data->sync_flags;
+
+		if (data->timeout_source) {
+			if (data->timeout_source != -1)
+				g_source_remove (data->timeout_source);
+			data->timeout_source = 0;
+		}
+
+		if (data->running_thread) {
+			GThread *thread = data->running_thread;
+
+			/* do not do anything later */
+			data = NULL;
+
+			/* unlock here, thus the thread can hold the lock again */
+			UnlockQueue ();
+
+			/* it's running at the moment, wait for a finish.
+			   it'll remove structure from a sync_queue too. */
+			g_thread_join (thread);
+		} else {
+			g_hash_table_remove (sync_queue, pFile);
+		}
+
+		if (data) {
+			static gboolean no_sync_on_close = FALSE, iKnow = FALSE;
+
+			if (!iKnow) {
+				iKnow = TRUE;
+				no_sync_on_close = getenv ("CAMEL_NO_SYNC_ON_CLOSE") != NULL;
+			}
+
+			/* do not block queue on while syncing */
+			UnlockQueue ();
+
+			if (!no_sync_on_close) {
+				/* sync on close */
+				call_old_file_Sync (pFile, sync_flags);
+			}
+
+			g_free (data);
+		}
+	} else {
+		UnlockQueue ();
+	}
+}
+
+#define def_subclassed(_nm, _params, _call)			\
+static int							\
+camel_sqlite3_file_ ## _nm _params				\
+{								\
+	struct CamelSqlite3File *cFile;				\
+								\
+	g_return_val_if_fail (old_vfs != NULL, SQLITE_ERROR);	\
+	g_return_val_if_fail (pFile != NULL, SQLITE_ERROR);	\
+								\
+	cFile = (struct CamelSqlite3File *) pFile;		\
+	g_return_val_if_fail (cFile->old_vfs_file->pMethods != NULL, SQLITE_ERROR);	\
+	return cFile->old_vfs_file->pMethods->_nm _call;	\
+}
+
+def_subclassed (xRead, (sqlite3_file *pFile, void *pBuf, int iAmt, sqlite3_int64 iOfst), (cFile->old_vfs_file, pBuf, iAmt, iOfst))
+def_subclassed (xWrite, (sqlite3_file *pFile, const void *pBuf, int iAmt, sqlite3_int64 iOfst), (cFile->old_vfs_file, pBuf, iAmt, iOfst))
+def_subclassed (xTruncate, (sqlite3_file *pFile, sqlite3_int64 size), (cFile->old_vfs_file, size))
+def_subclassed (xFileSize, (sqlite3_file *pFile, sqlite3_int64 *pSize), (cFile->old_vfs_file, pSize))
+def_subclassed (xLock, (sqlite3_file *pFile, int lockType), (cFile->old_vfs_file, lockType))
+def_subclassed (xUnlock, (sqlite3_file *pFile, int lockType), (cFile->old_vfs_file, lockType))
+#if SQLITE_VERSION_NUMBER < 3006000
+def_subclassed (xCheckReservedLock, (sqlite3_file *pFile), (cFile->old_vfs_file))
+#else
+def_subclassed (xCheckReservedLock, (sqlite3_file *pFile, int *pResOut), (cFile->old_vfs_file, pResOut))
+#endif
+def_subclassed (xFileControl, (sqlite3_file *pFile, int op, void *pArg), (cFile->old_vfs_file, op, pArg))
+def_subclassed (xSectorSize, (sqlite3_file *pFile), (cFile->old_vfs_file))
+def_subclassed (xDeviceCharacteristics, (sqlite3_file *pFile), (cFile->old_vfs_file))
+
+#undef def_subclassed
+
+static int
+camel_sqlite3_file_xClose (sqlite3_file *pFile)
+{
+	struct CamelSqlite3File *cFile;
+	int res;
+
+	g_return_val_if_fail (old_vfs != NULL, SQLITE_ERROR);
+	g_return_val_if_fail (pFile != NULL, SQLITE_ERROR);
+
+	dequeue_sync (pFile);
+
+	cFile = (struct CamelSqlite3File *) pFile;
+	if (cFile->old_vfs_file->pMethods)
+		res = cFile->old_vfs_file->pMethods->xClose (cFile->old_vfs_file);
+	else
+		res = SQLITE_OK;
+
+	g_free (cFile->old_vfs_file);
+	cFile->old_vfs_file = NULL;
+
+	return res;
+}
+
+static int 
+camel_sqlite3_file_xSync (sqlite3_file *pFile, int flags)
+{
+	g_return_val_if_fail (old_vfs != NULL, SQLITE_ERROR);
+	g_return_val_if_fail (pFile != NULL, SQLITE_ERROR);
+
+	queue_sync (pFile, flags);
+
+	return SQLITE_OK;
+}
+
+static int
+camel_sqlite3_vfs_xOpen (sqlite3_vfs *pVfs, const char *zPath, sqlite3_file *pFile, int flags, int *pOutFlags)
+{
+	static sqlite3_io_methods io_methods = {0};
+	struct CamelSqlite3File *cFile;
+	int res;
+
+	g_return_val_if_fail (old_vfs != NULL, -1);
+	g_return_val_if_fail (pFile != NULL, -1);
+
+	cFile = (struct CamelSqlite3File *)pFile;
+	cFile->old_vfs_file = g_malloc0 (old_vfs->szOsFile);
+
+	res = old_vfs->xOpen (old_vfs, zPath, cFile->old_vfs_file, flags, pOutFlags);
+
+	g_static_rec_mutex_lock (&only_once_lock);
+
+	/* cFile->old_vfs_file->pMethods is NULL when open failed for some reason,
+	   thus do not initialize our structure when do not know the version */
+	if (io_methods.xClose == NULL && cFile->old_vfs_file->pMethods) {
+		/* initialize our subclass function only once */
+		io_methods.iVersion = cFile->old_vfs_file->pMethods->iVersion;
+
+		#define use_subclassed(x) io_methods.x = camel_sqlite3_file_ ## x
+		use_subclassed (xClose);
+		use_subclassed (xRead);
+		use_subclassed (xWrite);
+		use_subclassed (xTruncate);
+		use_subclassed (xSync);
+		use_subclassed (xFileSize);
+		use_subclassed (xLock);
+		use_subclassed (xUnlock);
+		use_subclassed (xCheckReservedLock);
+		use_subclassed (xFileControl);
+		use_subclassed (xSectorSize);
+		use_subclassed (xDeviceCharacteristics);
+		#undef use_subclassed
+	}
+
+	g_static_rec_mutex_unlock (&only_once_lock);
+
+	cFile->parent.pMethods = &io_methods;
+
+	return res;
+}
+
+static void
+init_sqlite_vfs (void)
+{
+	static sqlite3_vfs vfs = { 0 };
+
+	g_static_rec_mutex_lock (&only_once_lock);
+	if (old_vfs) {
+		g_static_rec_mutex_unlock (&only_once_lock);
+		return;
+	}
+
+	old_vfs = sqlite3_vfs_find (NULL);
+	if (!old_vfs) {
+		g_static_rec_mutex_unlock (&only_once_lock);
+		g_return_if_fail (old_vfs != NULL);
+		return;
+	}
+
+	memcpy (&vfs, old_vfs, sizeof (sqlite3_vfs));
+
+	vfs.szOsFile = sizeof (struct CamelSqlite3File);
+	vfs.zName = "camel_sqlite3_vfs";
+	vfs.xOpen = camel_sqlite3_vfs_xOpen;
+
+	sqlite3_vfs_register (&vfs, 1);
+
+	g_static_rec_mutex_unlock (&only_once_lock);
+}
+
 #define d(x) if (camel_debug("sqlite")) x
 #define START(stmt) 	if (camel_debug("dbtime")) { g_print ("\n===========\nDB SQL operation [%s] started\n", stmt); if (!cdb->priv->timer) { cdb->priv->timer = g_timer_new (); } else { g_timer_reset(cdb->priv->timer);} }
 #define END 	if (camel_debug("dbtime")) { g_timer_stop (cdb->priv->timer); g_print ("DB Operation ended. Time Taken : %f\n###########\n", g_timer_elapsed (cdb->priv->timer, NULL)); }
+#define STARTTS(stmt) 	if (camel_debug("dbtimets")) { g_print ("\n===========\nDB SQL operation [%s] started\n", stmt); if (!cdb->priv->timer) { cdb->priv->timer = g_timer_new (); } else { g_timer_reset(cdb->priv->timer);} }
+#define ENDTS 	if (camel_debug("dbtimets")) { g_timer_stop (cdb->priv->timer); g_print ("DB Operation ended. Time Taken : %f\n###########\n", g_timer_elapsed (cdb->priv->timer, NULL)); }
+
 
 struct _CamelDBPrivate {
 	GTimer *timer;
@@ -92,6 +458,8 @@ camel_db_open (const char *path, CamelEx
 	char *cache;
 	int ret;
 
+	init_sqlite_vfs ();
+
 	CAMEL_DB_USE_SHARED_CACHE;
 	
 	ret = sqlite3_open(path, &db);
@@ -126,8 +494,13 @@ camel_db_open (const char *path, CamelEx
 		cache = g_strdup ("PRAGMA cache_size=100");
 
 	camel_db_command (cdb, cache, NULL);
-
 	g_free (cache);
+	if (g_getenv("CAMEL_SQLITE_IN_MEMORY") != NULL) {
+		/* Optionally turn off Journaling, this gets over fsync issues, but could be risky */
+		camel_db_command (cdb, "PRAGMA main.journal_mode = off", NULL);
+		camel_db_command (cdb, "PRAGMA temp_store = memory", NULL);
+	}
+
 
 	sqlite3_busy_timeout (cdb->db, CAMEL_DB_SLEEP_INTERVAL);
 
@@ -196,6 +569,8 @@ camel_db_begin_transaction (CamelDB *cdb
 		g_static_rec_mutex_lock (&trans_lock);
 
 	g_mutex_lock (cdb->lock);
+	STARTTS("BEGIN");
+
 	return (cdb_sql_exec (cdb->db, "BEGIN", ex));
 }
 
@@ -206,9 +581,9 @@ camel_db_end_transaction (CamelDB *cdb,
 	if (!cdb)
 		return -1;
 
-	START("COMMIT");
 	ret = cdb_sql_exec (cdb->db, "COMMIT", ex);
-	END;
+	ENDTS;
+
 	g_mutex_unlock (cdb->lock);
 	if (g_getenv("SQLITE_TRANSLOCK"))
 		g_static_rec_mutex_unlock (&trans_lock);
@@ -251,7 +626,7 @@ camel_db_transaction_command (CamelDB *c
 		return -1;
 
 	g_mutex_lock (cdb->lock);
-	START("BEGIN");
+	STARTTS("BEGIN");
 	ret = cdb_sql_exec (cdb->db, "BEGIN", ex);
 	if (ret)
 		goto end;
@@ -266,7 +641,7 @@ camel_db_transaction_command (CamelDB *c
 	}
 
 	ret = cdb_sql_exec (cdb->db, "COMMIT", ex);
-	END;
+	ENDTS;
 end:
 	g_mutex_unlock (cdb->lock);
 	return ret;
@@ -440,7 +815,7 @@ camel_db_count_total_message_info (Camel
 	if (!cdb)
 		return -1;
 	
-	query = sqlite3_mprintf ("SELECT COUNT (*) FROM %Q", table_name);
+	query = sqlite3_mprintf ("SELECT COUNT (*) FROM %Q where read=0 or read=1", table_name);
 
 	ret = camel_db_count_message_info (cdb, query, count, ex);
 	sqlite3_free (query);
@@ -563,6 +938,53 @@ camel_db_delete_uid_from_vfolder_transac
 	return ret;
 }
 
+struct _db_data_uids_flags {
+	GPtrArray *uids;
+	GPtrArray *flags;
+};
+static int
+read_uids_flags_callback (void *ref, int ncol, char ** cols, char ** name)
+{
+	struct _db_data_uids_flags *data= (struct _db_data_uids_flags *) ref;
+	 
+	int i;
+	for (i = 0; i < ncol; ++i) {
+		if (!strcmp (name [i], "uid"))
+			g_ptr_array_add (data->uids, (char *) (camel_pstring_strdup(cols [i])));
+		else if (!strcmp (name [i], "flags"))
+			g_ptr_array_add (data->flags, GUINT_TO_POINTER(strtoul (cols [i], NULL, 10)));
+	}
+	 
+	 return 0;
+}
+
+int
+camel_db_get_folder_uids_flags (CamelDB *db, char *folder_name, char *sort_by, char *collate, GPtrArray *summary, GHashTable *table, CamelException *ex)
+{
+	 GPtrArray *uids = summary;
+	 GPtrArray *flags = g_ptr_array_new ();
+	 char *sel_query;
+	 int ret;
+	 struct _db_data_uids_flags data;
+	 int i;
+	
+	 data.uids = uids;
+	 data.flags = flags;
+
+
+	 sel_query = sqlite3_mprintf("SELECT uid,flags FROM %Q%s%s%s%s", folder_name, sort_by ? " order by " : "", sort_by ? sort_by: "", (sort_by && collate) ? " collate " : "", (sort_by && collate) ? collate : "");	 
+
+	 ret = camel_db_select (db, sel_query, read_uids_flags_callback, &data, ex);
+	 sqlite3_free (sel_query);
+
+	 for (i=0; i<uids->len; i++) {
+		 g_hash_table_insert (table, uids->pdata[i], flags->pdata[i]);
+	 }
+
+	 g_ptr_array_free (flags, TRUE);
+	 return ret;
+}
+
 static int
 read_uids_callback (void *ref, int ncol, char ** cols, char ** name)
 {
@@ -792,7 +1214,7 @@ write_mir (CamelDB *cdb, const char *fol
 
 	/* NB: UGLIEST Hack. We can't modify the schema now. We are using msg_security (an unsed one to notify of FLAGGED/Dirty infos */
 
-	ins_query = sqlite3_mprintf ("INSERT INTO %Q VALUES (%Q, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %ld, %ld, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q )", 
+	ins_query = sqlite3_mprintf ("INSERT OR REPLACE INTO %Q VALUES (%Q, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %ld, %ld, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q )", 
 			folder_name, record->uid, record->flags,
 			record->msg_type, record->read, record->deleted, record->replied,
 			record->important, record->junk, record->attachment, record->dirty,
@@ -803,8 +1225,8 @@ write_mir (CamelDB *cdb, const char *fol
 			record->part, record->labels, record->usertags,
 			record->cinfo, record->bdata);
 
-	if (delete_old_record)
-			del_query = sqlite3_mprintf ("DELETE FROM %Q WHERE uid = %Q", folder_name, record->uid);
+//	if (delete_old_record)
+//			del_query = sqlite3_mprintf ("DELETE FROM %Q WHERE uid = %Q", folder_name, record->uid);
 
 #if 0
 	char *upd_query;
@@ -852,12 +1274,14 @@ camel_db_write_folder_info_record (Camel
 	g_free (upd_query);
 #else
 
-	ret = camel_db_add_to_transaction (cdb, del_query, ex);
+	//if (delete_old_record)
+	//		ret = camel_db_add_to_transaction (cdb, del_query, ex);
 	ret = camel_db_add_to_transaction (cdb, ins_query, ex);
 
 #endif
 
-	sqlite3_free (del_query);
+	//if (delete_old_record)
+	//		sqlite3_free (del_query);
 	sqlite3_free (ins_query);
 
 	return ret;
@@ -1009,6 +1433,9 @@ cdb_delete_ids (CamelDB *cdb, const char
 int
 camel_db_delete_uids (CamelDB *cdb, const char * folder_name, GSList *uids, CamelException *ex)
 {
+	if(!uids || !uids->data)
+		return 0;
+
 	return cdb_delete_ids (cdb, folder_name, uids, "", "uid", ex);
 }
 
--- camel/camel-db.h	2009-02-04 09:13:36.000000000 +0530
+++ camel/camel-db.h	2009-02-04 10:11:32.000000000 +0530
@@ -163,6 +163,7 @@ int camel_db_add_to_vfolder (CamelDB *db
 int camel_db_add_to_vfolder_transaction (CamelDB *db, char *folder_name, char *vuid, CamelException *ex);
 
 int camel_db_get_folder_uids (CamelDB *db, char *folder_name, char *sort_by, char *collate, GPtrArray *array, CamelException *ex);
+int camel_db_get_folder_uids_flags (CamelDB *db, char *folder_name, char *sort_by, char *collate, GPtrArray *summary, GHashTable *table, CamelException *ex);
 
 GPtrArray * camel_db_get_folder_junk_uids (CamelDB *db, char *folder_name, CamelException *ex);
 GPtrArray * camel_db_get_folder_deleted_uids (CamelDB *db, char *folder_name, CamelException *ex);
--- camel/camel-folder.c	2009-02-04 09:13:36.000000000 +0530
+++ camel/camel-folder.c	2009-02-04 10:11:32.000000000 +0530
@@ -1171,24 +1171,9 @@ get_uids(CamelFolder *folder)
 	GPtrArray *array;
 	int i, j, count;
 
-	array = g_ptr_array_new();
+	g_return_val_if_fail(folder->summary != NULL, g_ptr_array_new ());
 
-	g_return_val_if_fail(folder->summary != NULL, array);
-
-	count = camel_folder_summary_count(folder->summary);
-	g_ptr_array_set_size(array, count);
-	for (i = 0, j = 0; i < count; i++) {
-		CamelMessageInfo *info = camel_folder_summary_index(folder->summary, i);
-		
-		if (info) {
-			array->pdata[j++] = (char *)camel_pstring_strdup (camel_message_info_uid (info));
-			camel_message_info_free(info);
-		}
-	}
-	
-	g_ptr_array_set_size (array, j);
-	
-	return array;
+	return camel_folder_summary_array (folder->summary);
 }
 
 
--- camel/camel-folder-summary.c	2009-02-04 09:13:36.000000000 +0530
+++ camel/camel-folder-summary.c	2009-02-04 10:23:32.000000000 +0530
@@ -40,6 +40,7 @@
 
 #include <libedataserver/e-memory.h>
 
+#include "camel-debug.h"
 #include "camel-db.h"
 #include "camel-file-utils.h"
 #include "camel-folder-summary.h"
@@ -79,7 +80,6 @@ static pthread_mutex_t info_lock = PTHRE
 /* this should probably be conditional on it existing */
 #define USE_BSEARCH
 
-#define dd(x)
 #define d(x)
 #define io(x)			/* io debug */
 #define w(x)
@@ -155,6 +155,7 @@ camel_folder_summary_init (CamelFolderSu
 
 	s->message_info_size = sizeof(CamelMessageInfoBase);
 	s->content_info_size = sizeof(CamelMessageContentInfo);
+	p->flag_cache = g_hash_table_new (g_str_hash, g_str_equal);
 
 	s->message_info_chunks = NULL;
 	s->content_info_chunks = NULL;
@@ -200,7 +201,7 @@ camel_folder_summary_finalize (CamelObje
 	CamelFolderSummary *s = (CamelFolderSummary *)obj;
 
 	p = _PRIVATE(obj);
-
+	g_hash_table_destroy (p->flag_cache);
 	if (s->timeout_handle)
 		g_source_remove (s->timeout_handle);
 	//camel_folder_summary_clear(s);
@@ -834,7 +835,7 @@ remove_cache (CamelSession *session, Cam
 	CAMEL_SUMMARY_LOCK (s, summary_lock);
 	g_hash_table_foreach_remove  (s->loaded_infos, (GHRFunc) remove_item, s);
 	CAMEL_SUMMARY_UNLOCK (s, summary_lock);
-	d(printf("done .. now %d\n",g_hash_table_size (s->loaded_infos)));
+	dd(printf("done .. now %d\n",g_hash_table_size (s->loaded_infos)));
 
 	s->cache_load_time = time(NULL);
 	
@@ -915,8 +916,6 @@ camel_folder_summary_reload_from_db (Cam
 	if (!g_getenv("CAMEL_FREE_INFOS") && !s->timeout_handle) 
 		s->timeout_handle = g_timeout_add_seconds (SUMMARY_CACHE_DROP, (GSourceFunc) cfs_try_release_memory, s);
 
-	d(printf("Triggering summary_reloaded on %s %p\n", s->folder->full_name, s));
-	camel_object_trigger_event(s, "summary_reloaded", s);
 	return ret == 0 ? 0 : -1;
 }
 
@@ -931,14 +930,23 @@ camel_folder_summary_dump (CamelFolderSu
 	printf("\n");
 }
 
+GHashTable *
+camel_folder_summary_get_flag_cache (CamelFolderSummary *summary)
+{
+	struct _CamelFolderSummaryPrivate *p = _PRIVATE(summary);
+	
+	return p->flag_cache;
+}
+
 int
 camel_folder_summary_load_from_db (CamelFolderSummary *s, CamelException *ex)
 {
 	CamelDB *cdb;
 	char *folder_name;
 	int ret = 0;
+	struct _CamelFolderSummaryPrivate *p = _PRIVATE(s);
+
 	/* struct _db_pass_data data; */
-	
 	d(printf ("\ncamel_folder_summary_load_from_db called \n"));
 	s->flags &= ~CAMEL_SUMMARY_DIRTY;
 
@@ -950,7 +958,7 @@ camel_folder_summary_load_from_db (Camel
 	folder_name = s->folder->full_name;
 	cdb = s->folder->parent_store->cdb_r;
 
-	ret = camel_db_get_folder_uids (cdb, folder_name, (char *)s->sort_by, (char *)s->collate, s->uids, ex);
+	ret = camel_db_get_folder_uids_flags (cdb, folder_name, (char *)s->sort_by, (char *)s->collate, s->uids, p->flag_cache, ex);
 	/* camel_folder_summary_dump (s); */
 
 #if 0
@@ -1761,7 +1769,8 @@ camel_folder_summary_add (CamelFolderSum
 	#warning "FIXME: SHould we ref it or redesign it later on"
 	/* The uid array should have its own memory. We will unload the infos when not reqd.*/
 	g_ptr_array_add (s->uids, (gpointer) camel_pstring_strdup((camel_message_info_uid(info))));
-	
+	g_hash_table_replace (_PRIVATE(s)->flag_cache, (char *)info->uid, GUINT_TO_POINTER(camel_message_info_flags(info)));
+
 	g_hash_table_insert (s->loaded_infos, (gpointer) camel_message_info_uid (info), info);
 	s->flags |= CAMEL_SUMMARY_DIRTY;
 
@@ -1792,9 +1801,13 @@ camel_folder_summary_insert (CamelFolder
 		g_ptr_array_add (s->uids, (char *) camel_pstring_strdup(camel_message_info_uid(info)));
 	
 	g_hash_table_insert (s->loaded_infos, (char *) camel_message_info_uid (info), info);
+	if (load) {
+		g_hash_table_replace (_PRIVATE(s)->flag_cache, (char *)info->uid, GUINT_TO_POINTER(camel_message_info_flags(info)));
+	}
+
 	if (!load)
 		s->flags |= CAMEL_SUMMARY_DIRTY;
-
+	
 	CAMEL_SUMMARY_UNLOCK(s, summary_lock);	
 }
 
@@ -3102,7 +3115,7 @@ static CamelMessageInfo *
 message_info_load(CamelFolderSummary *s, FILE *in)
 {
 	CamelMessageInfoBase *mi;
-	guint count;
+	guint32 count;
 	int i;
 	char *subject, *from, *to, *cc, *mlist, *uid;
 
@@ -4667,7 +4680,8 @@ info_set_flags(CamelMessageInfo *info, g
 		if (((junk && !(mi->flags & CAMEL_MESSAGE_DELETED)))||  (deleted && !(mi->flags & CAMEL_MESSAGE_JUNK)) )
 			mi->summary->visible_count -= junk ? junk : deleted;
 	}
-
+	if (mi->uid)
+		g_hash_table_replace (_PRIVATE(mi->summary)->flag_cache, (char *)mi->uid, GUINT_TO_POINTER(mi->flags));
 	if (mi->summary && mi->summary->folder && mi->uid) {
 		CamelFolderChangeInfo *changes = camel_folder_change_info_new();
 
@@ -4682,7 +4696,11 @@ info_set_flags(CamelMessageInfo *info, g
 	return TRUE;
 }
 
-
+void
+camel_folder_summary_update_flag_cache (CamelFolderSummary *s, const char *uid, guint32 flag)
+{
+	g_hash_table_replace (_PRIVATE(s)->flag_cache, (char *) uid, GUINT_TO_POINTER(flag));	
+}
 /**
  * camel_message_info_set_flags:
  * @mi: a #CamelMessageInfo
@@ -4887,6 +4905,4 @@ camel_folder_summary_class_init (CamelFo
 
 	klass->info_set_flags = info_set_flags;
 	
-	camel_object_class_add_event(camel_object_class, "summary_reloaded", NULL);
-	
 }
--- camel/camel-folder-summary.h	2008-10-16 10:07:29.000000000 +0530
+++ camel/camel-folder-summary.h	2009-02-04 10:11:32.000000000 +0530
@@ -407,6 +407,8 @@ gboolean camel_folder_summary_check_uid
 GPtrArray *camel_folder_summary_array(CamelFolderSummary *summary);
 GHashTable *camel_folder_summary_get_hashtable(CamelFolderSummary *s);
 void camel_folder_summary_free_hashtable (GHashTable *ht);
+GHashTable *camel_folder_summary_get_flag_cache (CamelFolderSummary *summary);
+void camel_folder_summary_update_flag_cache (CamelFolderSummary *s, const char *uid, guint32 flag);
 
 /* basically like strings, but certain keywords can be compressed and de-cased */
 int camel_folder_summary_encode_token(FILE *out, const char *str);
--- camel/camel-private.h	2008-10-13 13:37:53.000000000 +0530
+++ camel/camel-private.h	2009-02-04 10:11:32.000000000 +0530
@@ -128,6 +128,7 @@ struct _CamelFolderSummaryPrivate {
 	GMutex *filter_lock;	/* for accessing any of the filtering/indexing stuff, since we share them */
 	GMutex *alloc_lock;	/* for setting up and using allocators */
 	GMutex *ref_lock;	/* for reffing/unreffing messageinfo's ALWAYS obtain before summary_lock */
+	GHashTable *flag_cache;
 };
 
 #define CAMEL_SUMMARY_LOCK(f, l) \
@@ -157,6 +158,7 @@ struct _CamelVeeFolderPrivate {
 	GMutex *summary_lock;		/* for locking vfolder summary */
 	GMutex *subfolder_lock;		/* for locking the subfolder list */
 	GMutex *changed_lock;		/* for locking the folders-changed list */
+	int unread_vfolder;
 };
 
 #define CAMEL_VEE_FOLDER_LOCK(f, l) \
--- camel/camel-search-sql-sexp.c	2009-02-04 09:13:36.000000000 +0530
+++ camel/camel-search-sql-sexp.c	2009-02-04 10:34:08.000000000 +0530
@@ -297,11 +297,11 @@ match_all(struct _ESExp *f, int argc, st
 	ESExpResult *r;
 
 	d(printf("executing match-all: %d", argc));
-	if (argv[0]->type != ESEXP_TERM_TIME)
+	if (argv[0]->type != ESEXP_TERM_BOOL)
 		r = e_sexp_term_eval(f, argv[0]);
 	else {
 		r = e_sexp_result_new(f, ESEXP_RES_STRING);
-		r->value.string = g_strdup("");
+		r->value.string = g_strdup(argv[0]->value.bool ? "1" : "0");
 	}
 
 	return r;
@@ -371,7 +371,7 @@ check_header (struct _ESExp *f, int argc
 					value = get_db_safe_string (tstr);
 					g_free (tstr);
 				}
-				str = g_strdup_printf("(%s LIKE %s)", headername, value);
+				str = g_strdup_printf("(%s IS NOT NULL AND %s LIKE %s)", headername, headername, value);
 				g_free(value);
 			}
 		}
@@ -562,7 +562,7 @@ sql_exp (struct _ESExp *f, int argc, str
 static struct {
 	char *name;
 	ESExpFunc *func;
-	int builtin :1;
+	int immediate :1;
 } symbols[] = {
 	{ "and", (ESExpFunc *) func_and, 1 },
 	{ "or", (ESExpFunc *) func_or, 1},
@@ -571,8 +571,8 @@ static struct {
 	{ ">", (ESExpFunc *)eval_gt, 1},
 	{ "<", (ESExpFunc *)eval_lt, 1},
 
-	{ "match-all", (ESExpFunc *)match_all, 0 },
-	{ "match-threads", (ESExpFunc *)match_threads, 0 },
+	{ "match-all", (ESExpFunc *)match_all, 1 },
+	{ "match-threads", (ESExpFunc *)match_threads, 1 },
 /* 	{ "body-contains", body_contains}, */ /* We don't store body on the db. */
 	{ "header-contains", header_contains, 0},
 	{ "header-matches", header_matches, 0},
@@ -603,8 +603,12 @@ camel_sexp_to_sql_sexp (const char *sql)
 	sexp = e_sexp_new();
 
 	for(i=0;i<sizeof(symbols)/sizeof(symbols[0]);i++) {
-		e_sexp_add_function(sexp, 0, symbols[i].name,
-				    symbols[i].func, NULL);
+		if (symbols[i].immediate)
+			e_sexp_add_ifunction(sexp, 0, symbols[i].name,
+					    symbols[i].func, NULL);
+		else
+			e_sexp_add_function(sexp, 0, symbols[i].name,
+					    symbols[i].func, NULL);
 	}
 
 	e_sexp_input_text (sexp, sql, strlen (sql));
--- camel/camel-vee-folder.c	2009-02-04 09:13:36.000000000 +0530
+++ camel/camel-vee-folder.c	2009-02-04 10:11:34.000000000 +0530
@@ -83,9 +83,6 @@ static void folder_changed(CamelFolder *
 static void subfolder_deleted(CamelFolder *f, void *event_data, CamelVeeFolder *vf);
 static void folder_renamed(CamelFolder *f, const char *old, CamelVeeFolder *vf);
 
-static void folder_changed_remove_uid(CamelFolder *sub, const char *uid, const char hash[8], int keep, CamelVeeFolder *vf);
-static void summary_reloaded(CamelObject *o, void *event_data, void *data);
-
 static CamelFolderClass *camel_vee_folder_parent;
 
 CamelType
@@ -150,7 +147,7 @@ camel_vee_folder_new(CamelStore *parent_
 		camel_vee_folder_construct(vf, parent_store, full, name, flags);
 	}
 
-	d(printf("returning folder %s %p, count = %d\n", name, vf, camel_folder_get_message_count((CamelFolder *)vf)));
+	d(printf("returning folder %s %p, count = %d\n", full, vf, camel_folder_get_message_count((CamelFolder *)vf)));
 
 	if (vf) {
 		tmp = g_strdup_printf("%s/%s.cmeta", ((CamelService *)parent_store)->url->path, full);
@@ -221,19 +218,12 @@ camel_vee_folder_add_folder(CamelVeeFold
 
 	d(printf("camel_vee_folder_add_folder(%s, %s)\n", ((CamelFolder *)vf)->full_name, sub->full_name));
 
-	cache = camel_folder_summary_cache_size(sub->summary);
-	if (!cache) {
-		camel_object_hook_event(sub->summary, "summary_reloaded", summary_reloaded, vf);
-		g_hash_table_insert(vf->loaded, sub, GINT_TO_POINTER(1));
-	}
 	camel_object_hook_event((CamelObject *)sub, "folder_changed", (CamelObjectEventHookFunc)folder_changed, vf);
 	camel_object_hook_event((CamelObject *)sub, "deleted", (CamelObjectEventHookFunc)subfolder_deleted, vf);
 	camel_object_hook_event((CamelObject *)sub, "renamed", (CamelObjectEventHookFunc)folder_renamed, vf);
 
 	((CamelVeeFolderClass *)((CamelObject *)vf)->klass)->add_folder(vf, sub);
 	
-	if (cache) 
-		summary_reloaded((CamelObject *) sub->summary, (void *)sub->summary, (void *)vf);
 }
 
 /**
@@ -266,10 +256,7 @@ camel_vee_folder_remove_folder(CamelVeeF
 	camel_object_unhook_event((CamelObject *)sub, "folder_changed", (CamelObjectEventHookFunc) folder_changed, vf);
 	camel_object_unhook_event((CamelObject *)sub, "deleted", (CamelObjectEventHookFunc) subfolder_deleted, vf);
 	camel_object_unhook_event((CamelObject *)sub, "renamed", (CamelObjectEventHookFunc) folder_renamed, vf);
-	if (GPOINTER_TO_INT(g_hash_table_lookup (vf->loaded, sub))) {
-		g_hash_table_remove (vf->loaded, sub);		
-		camel_object_unhook_event((CamelObject *)sub->summary, "summary_reloaded", (CamelObjectEventHookFunc) summary_reloaded, vf);
-	}
+
 	
 	p->folders = g_list_remove(p->folders, sub);
 	
@@ -516,6 +503,7 @@ summary_header_to_db (CamelFolderSummary
 	CamelFIRecord * record = g_new0 (CamelFIRecord, 1);
 	CamelDB *db;
 	char *table_name;
+	guint32 visible, unread, deleted, junked, junked_not_deleted;
 
 	/* We do this during write, so lets use write handle, though we gonna read */
 	db = s->folder->parent_store->cdb_w;
@@ -530,7 +518,14 @@ summary_header_to_db (CamelFolderSummary
 	record->time = s->time;
 
 	record->saved_count = s->uids->len;
-	if (!(((CamelVeeSummary *) s)->force_counts) && !g_getenv("FORCE_VFOLDER_COUNT")) {
+	camel_object_get(s->folder, NULL,
+				 CAMEL_FOLDER_DELETED, &deleted,
+				 CAMEL_FOLDER_VISIBLE, &visible,
+				 CAMEL_FOLDER_JUNKED, &junked,
+				 CAMEL_FOLDER_JUNKED_NOT_DELETED, &junked_not_deleted,
+				 CAMEL_FOLDER_UNREAD, &unread, NULL);
+	if (1) { /* We always would do this. Just refactor the code again. */ 
+		//!(((CamelVeeSummary *) s)->force_counts) && !g_getenv("FORCE_VFOLDER_COUNT")) {
 		/* We should be in sync always. so use the count. Don't search.*/
 		record->junk_count = s->junk_count;
 		record->deleted_count = s->deleted_count;
@@ -597,6 +592,7 @@ vee_sync(CamelFolder *folder, gboolean e
 
 		node = node->next;
 	}
+
 #if 0
 	/* Seems like we are doing something wrong with this, as folder_changed happens after this, the counts are misleading.
 	 * Anyways we do a force sync on exit, it should be all fine.
@@ -605,13 +601,37 @@ vee_sync(CamelFolder *folder, gboolean e
 	camel_db_write_folder_info_record (folder->parent_store->cdb, record, ex);
 	g_free (record);
 #endif	
+	/* It makes no sense to clear the folders_changed list without
+	 * actually rebuilding. */
+#if 0
 	if (node == NULL) {
 		CAMEL_VEE_FOLDER_LOCK(vf, changed_lock);
 		g_list_free(p->folders_changed);
 		p->folders_changed = NULL;
 		CAMEL_VEE_FOLDER_UNLOCK(vf, changed_lock);
 	}
-
+#endif
+	if (vf->priv->unread_vfolder == 1) {
+		/* Cleanup Junk/Trash uids */
+		int i, count;
+		count = folder->summary->uids->len;
+		GSList *del = NULL;
+
+		for (i=0; i < count; i++) {
+			CamelVeeMessageInfo *mi = camel_folder_summary_index (folder->summary, i);
+			if (mi->old_flags & CAMEL_MESSAGE_DELETED) {
+				del = g_slist_prepend (del, (gpointer) camel_pstring_strdup(((CamelMessageInfo *)mi)->uid));
+				camel_folder_summary_remove_index_fast (folder->summary, i);
+				count--;
+				i--;
+
+			}
+			camel_message_info_free (mi);
+		}
+		camel_db_delete_vuids (folder->parent_store->cdb_w, folder->full_name, "", del, ex);
+		g_slist_foreach (del, (GFunc) camel_pstring_free, NULL);
+		g_slist_free (del);			
+	}
 	CAMEL_VEE_FOLDER_UNLOCK(vf, subfolder_lock);
 
 	camel_object_state_write(vf);
@@ -839,8 +859,21 @@ static void vee_delete(CamelFolder *fold
 /* ********************************************************************** *
    utility functions */
 
-#define COUNT_ADD_EXPR(lval, sexpr) expr = g_strdup_printf("and (%s %s)", vf->expression, sexpr); lval += count_folder(f, expr, ex); g_free(expr);
-#define COUNT_DEL_EXPR(lval, sexpr) expr = g_strdup_printf("and (%s %s)", vf->expression, sexpr); lval -= count_folder(ssummary->folder, expr, NULL); g_free(expr);
+/* A "correlating" expression has the property that whether a message matches
+ * depends on the other messages being searched.  folder_changed_change on a
+ * vfolder with a correlating expression may not make all the necessary updates,
+ * so the query is redone on the entire changed source folder the next time
+ * the vfolder is opened.
+ * 
+ * The only current example of a correlating expression is one that uses
+ * "match-threads". */
+static gboolean
+expression_is_correlating(const char *expr)
+{
+	/* XXX: Actually parse the expression to avoid triggering on
+	 * "match-threads" in the text the user is searching for! */
+	return (strstr(expr, "match-threads") != NULL);
+}
 
 /* must be called with summary_lock held */
 static CamelVeeMessageInfo *
@@ -865,7 +898,6 @@ vee_folder_remove_folder(CamelVeeFolder
 	GHashTable *unmatched_uids = vf->parent_vee_store ? vf->parent_vee_store->unmatched_uids : NULL;
 	CamelFolderSummary *ssummary = source->summary;
 	int killun = FALSE;
-	char *expr;
 	
 	if (vf == folder_unmatched)
 		return;
@@ -966,12 +998,6 @@ vee_folder_remove_folder(CamelVeeFolder
 
 	if (last != -1)
 		camel_folder_summary_remove_range(folder->summary, start, last);
-	
-	COUNT_DEL_EXPR(folder->summary->junk_count, "(match-all (system-flag  \"junk\"))");
-	COUNT_DEL_EXPR(folder->summary->deleted_count, "(match-all (system-flag  \"deleted\"))");
-	COUNT_DEL_EXPR(folder->summary->unread_count, "(match-all (not (system-flag  \"Seen\")))");
-	COUNT_DEL_EXPR(folder->summary->visible_count, "(match-all (and (not (system-flag \"deleted\")) (not (system-flag \"junk\"))))");
-	COUNT_DEL_EXPR(folder->summary->junk_not_deleted_count, "(match-all (and (not (system-flag \"deleted\")) (system-flag \"junk\")))");
 				
 	if (folder_unmatched) {
 		if (camel_folder_change_info_changed(folder_unmatched->changes)) {
@@ -1006,7 +1032,7 @@ struct _update_data {
 	char hash[8];
 	CamelVeeFolder *folder_unmatched;
 	GHashTable *unmatched_uids;
-	gboolean rebuilt;
+	gboolean rebuilt, correlating;
 };
 
 static void
@@ -1044,7 +1070,7 @@ folder_added_uid(char *uidin, void *valu
 		camel_folder_change_info_add_uid(u->vf->changes, camel_message_info_uid(mi));
 		#warning "Handle exceptions"
 		#warning "Make all these as transactions, just testing atm"
-		if (u->rebuilt)
+		if (u->rebuilt && !u->correlating)
 			camel_db_add_to_vfolder_transaction (((CamelFolder *) u->vf)->parent_store->cdb_w, ((CamelFolder *) u->vf)->full_name, (char *) camel_message_info_uid(mi), NULL);
 		if (!CAMEL_IS_VEE_FOLDER(u->source) && u->unmatched_uids != NULL) {
 			if (g_hash_table_lookup_extended(u->unmatched_uids, camel_message_info_uid(mi), (void **)&oldkey, &oldval)) {
@@ -1075,6 +1101,11 @@ vee_rebuild_folder(CamelVeeFolder *vf, C
 	CamelFolderSummary *ssummary = source->summary;
 	gboolean rebuilded = FALSE;
 	char *shash; 
+
+	/* Since the source of a correlating vfolder has to be requeried in
+	 * full every time it changes, caching the results in the db is not
+	 * worth the effort.  Thus, DB use is conditioned on !correlating. */
+	gboolean correlating = expression_is_correlating(vf->expression);
 	
 	if (vf == folder_unmatched)
 		return 0;
@@ -1088,30 +1119,30 @@ vee_rebuild_folder(CamelVeeFolder *vf, C
 	if (vf->expression == NULL) {
 		match = g_ptr_array_new();
 	} else {
-		/* Load the folder results from the DB. */
-		match = camel_vee_summary_get_ids ((CamelVeeSummary *)folder->summary, u.hash);
-		d(printf("len = %d %d for %s %s\n", match ? match->len:0, rebuilded, source->full_name, shash));
-		if (!match) {
-			char *expr;
+		if (!correlating) {
+			/* Load the folder results from the DB. */
+			match = camel_vee_summary_get_ids ((CamelVeeSummary *)folder->summary, u.hash);
+		}
+		if (correlating ||
+			/* We take this to mean the results have not been cached.
+			 * XXX: It will also trigger if the result set is empty. */
+			match == NULL) {
 			match = camel_folder_search_by_expression(f, vf->expression, ex);
-			if (match == NULL)
+			if (match == NULL) /* Search failed */
 				return 0;
 			rebuilded = TRUE;
-			/* Update the counts */
-			COUNT_ADD_EXPR(folder->summary->junk_count, "(match-all (system-flag  \"junk\"))");
-			COUNT_ADD_EXPR(folder->summary->deleted_count, "(match-all (system-flag  \"deleted\"))");
-			COUNT_ADD_EXPR(folder->summary->unread_count, "(match-all (not (system-flag  \"Seen\")))");
-			COUNT_ADD_EXPR(folder->summary->visible_count, "(match-all (and (not (system-flag \"deleted\")) (not (system-flag \"junk\"))))");
-			COUNT_ADD_EXPR(folder->summary->junk_not_deleted_count, "(match-all (and (not (system-flag \"deleted\")) (system-flag \"junk\")))");
 		}
 		
 	}
+	dd(printf("vee_rebuild_folder(%s <- %s %s): match %d, correlating %d, rebuilded %d\n",
+		folder->full_name, source->full_name, shash, match->len, correlating, rebuilded));
 
 	u.source = source;
 	u.vf = vf;
 	u.folder_unmatched = folder_unmatched;
 	u.unmatched_uids = unmatched_uids;
 	u.rebuilt = rebuilded;
+	u.correlating = correlating;
 
 	CAMEL_VEE_FOLDER_LOCK(vf, summary_lock);
 
@@ -1190,13 +1221,13 @@ vee_rebuild_folder(CamelVeeFolder *vf, C
 		camel_folder_summary_remove_range(folder->summary, start, last);
 
 	/* now matchhash contains any new uid's, add them, etc */
-	if (rebuilded) {
+	if (rebuilded && !correlating) {
 		camel_db_begin_transaction (folder->parent_store->cdb_w, NULL);
 
 	}
 	g_hash_table_foreach(matchhash, (GHFunc)folder_added_uid, &u);
 
-	if (rebuilded)
+	if (rebuilded && !correlating)
 		camel_db_end_transaction (folder->parent_store->cdb_w, NULL);
 	
 	if (folder_unmatched != NULL) {
@@ -1242,7 +1273,8 @@ vee_rebuild_folder(CamelVeeFolder *vf, C
 	
 	/* Del the unwanted things from the summary, we don't hold any locks now. */
 	if (del_list) {
-		camel_db_delete_vuids(folder->parent_store->cdb_w, folder->full_name, shash, del_list, NULL);
+		if (!correlating)
+			camel_db_delete_vuids(folder->parent_store->cdb_w, folder->full_name, shash, del_list, NULL);
 		((CamelVeeSummary *)folder->summary)->force_counts = TRUE;
 		g_slist_foreach (del_list, (GFunc) camel_pstring_free, NULL);
 		g_slist_free (del_list);	
@@ -1274,95 +1306,10 @@ vee_rebuild_folder(CamelVeeFolder *vf, C
 	return 0;
 }
 
-/*
-
-  (match-folder "folder1" "folder2")
-
- */
-
-static void
-update_summary (CamelVeeMessageInfo *mi, guint32 flags, guint32 oldflags, gboolean add, gboolean use_old)
-{
-	int unread=0, deleted=0, junk=0;
-	CamelFolderSummary *summary = ((CamelMessageInfo *) mi)->summary;
-	
-	if (!(flags & CAMEL_MESSAGE_SEEN) && !(flags & CAMEL_MESSAGE_JUNK))
-		unread = 1;
-	
-	if (flags & CAMEL_MESSAGE_DELETED)
-		deleted = 1;
-
-	if (flags & CAMEL_MESSAGE_JUNK)
-		junk = 1;
-
-	d(printf("folder: %s %d %d: %p:%s\n", summary->folder->full_name, oldflags, add, mi, ((CamelMessageInfo *)mi)->uid));
-	d(printf("%d %d %d\n%d %d %d\n", !(flags & CAMEL_MESSAGE_SEEN), flags & CAMEL_MESSAGE_DELETED, flags & CAMEL_MESSAGE_JUNK, !(oldflags & CAMEL_MESSAGE_SEEN), oldflags & CAMEL_MESSAGE_DELETED, oldflags & CAMEL_MESSAGE_JUNK));
-	
-	if (!use_old) {
-		if (add) {
-			if (unread)
-				summary->unread_count += unread;
-			if (deleted)
-				summary->deleted_count += deleted;
-			if (junk)
-				summary->junk_count += junk;
-			if (junk && !deleted)
-				summary->junk_not_deleted_count += junk;
-			summary->visible_count++;
-			if (junk ||  deleted) 
-				summary->visible_count -= junk ? junk : deleted;
-
-			summary->saved_count++;		
-		} else  {
-			oldflags = use_old ? oldflags : flags;
-			unread = deleted = junk = 0;
-			if (!(oldflags & CAMEL_MESSAGE_SEEN) && !(oldflags & CAMEL_MESSAGE_JUNK))
-				unread -= 1;
-
-			if (oldflags & CAMEL_MESSAGE_DELETED)
-				deleted -= 1;
-
-			if (oldflags & CAMEL_MESSAGE_JUNK)
-				junk -= 1;			
-
-			if (unread)
-				summary->unread_count += unread;
-			if (deleted)
-				summary->deleted_count += deleted;
-			if (junk)
-				summary->junk_count += junk;
-			if (junk && !deleted)
-				summary->junk_not_deleted_count += junk;
-			if (!junk && !deleted)
-				summary->visible_count--;
-
-			summary->saved_count--;		
-		}
-	} else {
-		if (!(oldflags & CAMEL_MESSAGE_SEEN) && !(oldflags & CAMEL_MESSAGE_JUNK))
-			unread -= 1;
-
-		if (oldflags & CAMEL_MESSAGE_DELETED)
-			deleted -= 1;
-
-		if (oldflags & CAMEL_MESSAGE_JUNK)
-			junk -= 1;
-		if (unread)
-			summary->unread_count += unread;
-		if (deleted)
-			summary->deleted_count += deleted;
-		if (junk)
-			summary->junk_count += junk;
-		if (junk && !deleted)
-			summary->junk_not_deleted_count += junk;
-		if (junk ||  deleted) 
-			summary->visible_count -= junk ? junk : deleted;
-	}
-}
 		
 /* Hold all these with summary lock and unmatched summary lock held */
 static void
-folder_changed_add_uid(CamelFolder *sub, const char *uid, const char hash[8], CamelVeeFolder *vf)
+folder_changed_add_uid(CamelFolder *sub, const char *uid, const char hash[8], CamelVeeFolder *vf, gboolean use_db)
 {
 	CamelFolder *folder = (CamelFolder *)vf;	
 	CamelVeeMessageInfo *vinfo;
@@ -1378,10 +1325,9 @@ folder_changed_add_uid(CamelFolder *sub,
 		return;
 	
 	vuid = camel_message_info_uid(vinfo);
-	camel_db_add_to_vfolder_transaction (folder->parent_store->cdb_w, folder->full_name, (char *)vuid, NULL);
+	if (use_db)
+		camel_db_add_to_vfolder_transaction (folder->parent_store->cdb_w, folder->full_name, (char *)vuid, NULL);
 	camel_folder_change_info_add_uid(vf->changes,  vuid);
-	/* old flags and new flags should  be same, since we sync all times  */
-	update_summary (vinfo, camel_message_info_flags(vinfo), 0, TRUE, FALSE);
 	if ((vf->flags & CAMEL_STORE_FOLDER_PRIVATE) == 0 && !CAMEL_IS_VEE_FOLDER(sub) && folder_unmatched != NULL) {
 		if (g_hash_table_lookup_extended(unmatched_uids, vuid, (void **)&oldkey, &oldval)) {
 			n = GPOINTER_TO_INT (oldval);
@@ -1399,7 +1345,7 @@ folder_changed_add_uid(CamelFolder *sub,
 }
 
 static void
-folder_changed_remove_uid(CamelFolder *sub, const char *uid, const char hash[8], int keep, CamelVeeFolder *vf)
+folder_changed_remove_uid(CamelFolder *sub, const char *uid, const char hash[8], int keep, CamelVeeFolder *vf, gboolean use_db)
 {
 	CamelFolder *folder = (CamelFolder *)vf;
 	char *vuid, *oldkey;
@@ -1413,14 +1359,10 @@ folder_changed_remove_uid(CamelFolder *s
 	memcpy(vuid, hash, 8);
 	strcpy(vuid+8, uid);
 
-	vinfo = (CamelVeeMessageInfo *) camel_folder_summary_uid (((CamelFolder *) vf)->summary, vuid);
-	if (vinfo) {
-		update_summary (vinfo, vinfo->old_flags, 0, FALSE, FALSE);
-		camel_message_info_free((CamelMessageInfo *)vinfo);
-	}
 	camel_folder_change_info_remove_uid(vf->changes, vuid);
-        /* FIXME[disk-summary] Handle exception */
-	camel_db_delete_uid_from_vfolder_transaction (folder->parent_store->cdb_w, folder->full_name, vuid, NULL);
+	if (use_db)
+		/* FIXME[disk-summary] Handle exception */
+		camel_db_delete_uid_from_vfolder_transaction (folder->parent_store->cdb_w, folder->full_name, vuid, NULL);
 	camel_folder_summary_remove_uid_fast(folder->summary, vuid);
 
 	if ((vf->flags & CAMEL_STORE_FOLDER_PRIVATE) == 0 && !CAMEL_IS_VEE_FOLDER(sub) && folder_unmatched != NULL) {
@@ -1456,7 +1398,7 @@ folder_changed_remove_uid(CamelFolder *s
 }
 
 static void
-folder_changed_change_uid(CamelFolder *sub, const char *uid, const char hash[8], CamelVeeFolder *vf)
+folder_changed_change_uid(CamelFolder *sub, const char *uid, const char hash[8], CamelVeeFolder *vf, gboolean use_db)
 {
 	char *vuid;
 	CamelVeeMessageInfo *vinfo, *uinfo = NULL;
@@ -1475,22 +1417,19 @@ folder_changed_change_uid(CamelFolder *s
 		info = camel_folder_get_message_info(sub, uid);
 		if (info) {
 			if (vinfo) {
-				guint32 of = vinfo->old_flags;
 				camel_folder_change_info_change_uid(vf->changes, vuid);
-				update_summary (vinfo, camel_message_info_flags(info), of, FALSE /* Doesn't matter */, TRUE);
 				camel_message_info_free((CamelMessageInfo *)vinfo);
 			}
 
 			if (uinfo) {
 				camel_folder_change_info_change_uid(folder_unmatched->changes, vuid);
-				update_summary (uinfo, camel_message_info_flags(info), uinfo->old_flags, FALSE /* Doesn't matter */, TRUE);				
 				camel_message_info_free((CamelMessageInfo *)uinfo);
 			}
 
 			camel_folder_free_message_info(sub, info);
 		} else {
 			if (vinfo) {
-				folder_changed_remove_uid(sub, uid, hash, FALSE, vf);
+				folder_changed_remove_uid(sub, uid, hash, FALSE, vf, use_db);
 				camel_message_info_free((CamelMessageInfo *)vinfo);
 			}
 			if (uinfo)
@@ -1529,6 +1468,9 @@ folder_changed_change(CamelSession *sess
 	GHashTable *unmatched_uids = vf->parent_vee_store ? vf->parent_vee_store->unmatched_uids : NULL;
 	GPtrArray *present = NULL;
 
+	/* See vee_rebuild_folder. */
+	gboolean correlating = expression_is_correlating(vf->expression);
+
 	/* Check the folder hasn't beem removed while we weren't watching */
 	CAMEL_VEE_FOLDER_LOCK(vf, subfolder_lock);
 	if (g_list_find(_PRIVATE(vf)->folders, sub) == NULL) {
@@ -1598,7 +1540,7 @@ folder_changed_change(CamelSession *sess
 	/* Always remove removed uid's, in any case */
 	for (i=0;i<changes->uid_removed->len;i++) {
 		dd(printf("  removing uid '%s'\n", (char *)changes->uid_removed->pdata[i]));
-		folder_changed_remove_uid(sub, changes->uid_removed->pdata[i], hash, FALSE, vf);
+		folder_changed_remove_uid(sub, changes->uid_removed->pdata[i], hash, FALSE, vf, !correlating);
 	}
 
 	/* Add any newly matched or to unmatched folder if they dont */
@@ -1612,7 +1554,7 @@ folder_changed_change(CamelSession *sess
 			uid = changes->uid_added->pdata[i];
 			if (g_hash_table_lookup(matches_hash, uid)) {
 				dd(printf("  adding uid '%s' [newly matched]\n", (char *)uid));
-				folder_changed_add_uid(sub, uid, hash, vf);
+				folder_changed_add_uid(sub, uid, hash, vf, !correlating);
 			} else if ((vf->flags & CAMEL_STORE_FOLDER_PRIVATE) == 0) {
 				if (strlen(uid)+9 > vuidlen) {
 					vuidlen = strlen(uid)+64;
@@ -1638,20 +1580,31 @@ folder_changed_change(CamelSession *sess
 
 	/* Change any newly changed */
 	if (always_changed) {
-		GHashTable *ht_present = g_hash_table_new (g_str_hash, g_str_equal);
-
-		for (i=0;present && i<present->len;i++) {
-			folder_changed_change_uid(sub, present->pdata[i], hash, vf);
-			g_hash_table_insert (ht_present, present->pdata[i], present->pdata[i]);
-		}
-		
-		for (i=0; i<always_changed->len; i++) {
-			if (!present || !g_hash_table_lookup(ht_present, always_changed->pdata[i])) {
-				folder_changed_remove_uid(sub, always_changed->pdata[i], hash, FALSE, vf);
+		if (correlating) {
+			/* Messages may be pulled in by the correlation even if
+			 * they do not match the expression individually, so it
+			 * would be wrong to preemptively remove anything here.
+			 * vee_rebuild_folder will make any necessary removals
+			 * when it re-queries the entire source folder. */
+			for (i=0;i<always_changed->len;i++)
+				folder_changed_change_uid(sub, always_changed->pdata[i], hash, vf, !correlating);
+		} else {
+			GHashTable *ht_present = g_hash_table_new (g_str_hash, g_str_equal);
+	
+			for (i=0;present && i<present->len;i++) {
+				folder_changed_change_uid(sub, present->pdata[i], hash, vf, !correlating);
+				g_hash_table_insert (ht_present, present->pdata[i], present->pdata[i]);
+			}
+			
+			for (i=0; i<always_changed->len; i++) {
+				if (!present || !g_hash_table_lookup(ht_present, always_changed->pdata[i]))
+					/* XXX: IIUC, these messages haven't been deleted from the
+					 * source folder, so shouldn't "keep" be set to TRUE? */
+					folder_changed_remove_uid(sub, always_changed->pdata[i], hash, TRUE, vf, !correlating);
 			}
+	
+			g_hash_table_destroy (ht_present);
 		}
-
-		g_hash_table_destroy (ht_present);
 		g_ptr_array_free(always_changed, TRUE);
 	}
 
@@ -1679,21 +1632,21 @@ folder_changed_change(CamelSession *sess
 				if (g_hash_table_lookup(matches_hash, uid)) {
 					/* A uid we dont have, but now it matches, add it */
 					dd(printf("  adding uid '%s' [newly matched]\n", uid));
-					folder_changed_add_uid(sub, uid, hash, vf);
+					folder_changed_add_uid(sub, uid, hash, vf, !correlating);
 				} else {
 					/* A uid we still don't have, just change it (for unmatched) */
-					folder_changed_change_uid(sub, uid, hash, vf);
+					folder_changed_change_uid(sub, uid, hash, vf, !correlating);
 				}
 			} else {
 				if ((vf->flags & CAMEL_STORE_VEE_FOLDER_AUTO) == 0
 				    || g_hash_table_lookup(matches_hash, uid)) {
 					/* still match, or we're not auto-updating, change event, (if it changed) */
 					dd(printf("  changing uid '%s' [still matches]\n", uid));
-					folder_changed_change_uid(sub, uid, hash, vf);
+					folder_changed_change_uid(sub, uid, hash, vf, !correlating);
 				} else {
 					/* No longer matches, remove it, but keep it in unmatched (potentially) */
 					dd(printf("  removing uid '%s' [did match]\n", uid));
-					folder_changed_remove_uid(sub, uid, hash, TRUE, vf);
+					folder_changed_remove_uid(sub, uid, hash, TRUE, vf, !correlating);
 				}
 				camel_message_info_free((CamelMessageInfo *)vinfo);
 			}
@@ -1702,7 +1655,7 @@ folder_changed_change(CamelSession *sess
 	} else {
 		/* stuff didn't match but it changed - check unmatched folder for changes */
 		for (i=0;i<changed->len;i++)
-			folder_changed_change_uid(sub, changed->pdata[i], hash, vf);
+			folder_changed_change_uid(sub, changed->pdata[i], hash, vf, !correlating);
 	}
 
 	if (folder_unmatched != NULL) {
@@ -1745,15 +1698,25 @@ folder_changed_change(CamelSession *sess
 		camel_folder_change_info_free(unmatched_changes);
 	}
 	
-	if (vf_changes) {
-		/* If not auto-updating, keep track of changed folders for later re-sync */
-		if ((vf->flags & CAMEL_STORE_VEE_FOLDER_AUTO) == 0) {
-			CAMEL_VEE_FOLDER_LOCK(vf, changed_lock);
-			if (g_list_find(vf->priv->folders_changed, sub) != NULL)
-				vf->priv->folders_changed = g_list_prepend(vf->priv->folders_changed, sub);
-			CAMEL_VEE_FOLDER_UNLOCK(vf, changed_lock);
-		}
+	/* Add to folders_changed if we need to call vee_rebuild_folder, which
+	 * could be the case for two reasons:
+	 * - We changed the vfolder and it is not auto-updating.  Need to re-sync.
+	 * - Vfolder is correlating.  Changes to non-matching source messages
+	 *   won't be processed here and won't show up in vf_changes but may
+	 *   still affect the vfolder contents (e.g., non-matching messages
+	 *   added to a matching thread), so we re-run the query on the whole
+	 *   source folder.  (For match-threads, it may be enough to do this if
+	 *   changes->uid_added->len > 0, but I'm not completely sure and I'd
+	 *   rather be safe than sorry.)
+	 */
+	if ((vf_changes && (vf->flags & CAMEL_STORE_VEE_FOLDER_AUTO) == 0) || correlating) {
+		CAMEL_VEE_FOLDER_LOCK(vf, changed_lock);
+		if (g_list_find(vf->priv->folders_changed, sub) == NULL)
+			vf->priv->folders_changed = g_list_prepend(vf->priv->folders_changed, sub);
+		CAMEL_VEE_FOLDER_UNLOCK(vf, changed_lock);
+	}
 
+	if (vf_changes) {
 		camel_object_trigger_event((CamelObject *)vf, "folder_changed", vf_changes);
 		camel_folder_change_info_free(vf_changes);
 	}
@@ -1937,95 +1900,6 @@ vee_thaw(CamelFolder *folder)
 	CAMEL_FOLDER_CLASS (camel_vee_folder_parent)->thaw(folder);
 }
 
-
-struct _folder_flags_msg {
-	CamelSessionThreadMsg msg;
-	CamelFolder *sub;
-	CamelVeeFolder *vf;
-};
-
-static void
-folder_load_flags(CamelSession *session, CamelSessionThreadMsg *msg)
-{
-	struct _folder_flags_msg *m = (struct _folder_flags_msg *)msg;
-	CamelFolder *sub = m->sub;
-	CamelFolder *folder = (CamelFolder *)m->vf;
-	GPtrArray *array;
-	char *shash, hash[8];
-	int i;
-
-	camel_vee_folder_hash_folder(sub, hash);	
-	shash = g_strdup_printf("%c%c%c%c%c%c%c%c", hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7]);
-	dd(printf("Loading summary of %s to vfolder %s\n", sub->full_name, folder->full_name));
-
-	/* Get the summary of vfolder */
-	array = camel_folder_summary_array (folder->summary);
-	for (i=0; i<array->len; i++) {
-		if (strncmp(array->pdata[i], shash, 8) == 0) {
-			/* We have got a vuid for this sub folder.*/
-			CamelVeeMessageInfo *vinfo;
-			CamelMessageInfo *info;
-			char *uid;
-			vinfo = (CamelVeeMessageInfo *) camel_folder_summary_uid (folder->summary, (char *)array->pdata[i]);
-			if (!vinfo) /* What else we can do ?*/
-				continue;
-			uid = ((char *)array->pdata[i])+8;
-			info = camel_folder_summary_uid (sub->summary, uid);
-			if (!info) {
-				camel_message_info_free((CamelMessageInfo *)vinfo);
-				continue;
-			}
-
-			vinfo->old_flags = camel_message_info_flags (info);
-			camel_message_info_free((CamelMessageInfo *)info);
-			camel_message_info_free((CamelMessageInfo *)vinfo);
-		}
-	}
-	camel_folder_free_summary (folder, array);
-	g_free(shash);	
-}
-
-static void
-folder_load_flags_free(CamelSession *session, CamelSessionThreadMsg *msg)
-{
-	struct _folder_flags_msg *m = (struct _folder_flags_msg *)msg;
-	
-	camel_object_unref((CamelObject *)m->vf);
-	camel_object_unref((CamelObject *)m->sub);	
-}
-
-static CamelSessionThreadOps folder_flags_ops = {
-	folder_load_flags,
-	folder_load_flags_free,
-};
-
-static void
-summary_reloaded(CamelObject *o, void *event_data, void *data)
-{
-	CamelFolderSummary *summary = event_data;
-	CamelVeeFolder *vf = (CamelVeeFolder *)data;
-	struct _CamelVeeFolderPrivate *p = _PRIVATE(vf);	
-	struct _folder_flags_msg *m;
-	CamelSession *session = ((CamelService *)((CamelFolder *)vf)->parent_store)->session;
-
-	if (p->destroyed)
-		return; 
-
-	/* Kick off a thread to reload flags from the summary */
-	if (GPOINTER_TO_INT(g_hash_table_lookup (vf->loaded, summary->folder))) {	
-		g_hash_table_remove (vf->loaded, summary->folder);
-		camel_object_unhook_event((CamelObject *)o, "summary_reloaded", (CamelObjectEventHookFunc) summary_reloaded, data);
-	}
-	
-	m = camel_session_thread_msg_new(session, &folder_flags_ops, sizeof(*m));
-	m->sub = summary->folder;
-	camel_object_ref((CamelObject *)summary->folder);
-	m->vf = vf;
-	camel_object_ref((CamelObject *)vf);
-	camel_session_thread_queue(session, &m->msg, 0);
-	
-}
-
 /* vfolder base implementaitons */
 static void
 vee_add_folder(CamelVeeFolder *vf, CamelFolder *sub)
@@ -2098,6 +1972,106 @@ vee_set_expression(CamelVeeFolder *vf, c
 	CAMEL_VEE_FOLDER_UNLOCK(vf, subfolder_lock);
 }
 
+/* This entire code will be useless, since we sync the counts always. */
+static int
+vf_getv(CamelObject *object, CamelException *ex, CamelArgGetV *args)
+{
+	CamelFolder *folder = (CamelFolder *)object;
+	CamelVeeFolder *vf = (CamelVeeFolder *)folder;
+	int i;
+	guint32 tag;
+	int unread = -1, deleted = 0, junked = 0, visible = 0, count = -1, junked_not_deleted = -1;
+
+
+	for (i=0;i<args->argc;i++) {
+		CamelArgGet *arg = &args->argv[i];
+
+		tag = arg->tag;
+
+		/* NB: this is a copy of camel-folder.c with the unread count logic altered.
+		   makes sure its still atomically calculated */
+		switch (tag & CAMEL_ARG_TAG) {
+		case CAMEL_FOLDER_ARG_UNREAD:
+		case CAMEL_FOLDER_ARG_DELETED:
+		case CAMEL_FOLDER_ARG_JUNKED:
+		case CAMEL_FOLDER_ARG_JUNKED_NOT_DELETED:	
+		case CAMEL_FOLDER_ARG_VISIBLE:
+
+			if (vf->expression && vf->priv->unread_vfolder == -1)
+				camel_vee_summary_load_check_unread_vfolder ((CamelVeeSummary *)folder->summary);
+		
+			/* This is so we can get the values atomically, and also so we can calculate them only once */
+			if (unread == -1) {
+				int j;
+				CamelMessageInfoBase *info;
+				CamelVeeMessageInfo *vinfo;
+
+				unread = deleted = visible = junked = junked_not_deleted = 0;
+				count = camel_folder_summary_count(folder->summary);
+				for (j=0; j<count; j++) {
+					if ((info = (CamelMessageInfoBase *) camel_folder_summary_index(folder->summary, j))) {
+						guint32 flags;
+
+						vinfo = (CamelVeeMessageInfo *) info;
+						flags = vinfo->old_flags;// ? vinfo->old_flags : camel_message_info_flags(info);
+
+						if ((flags & (CAMEL_MESSAGE_SEEN)) == 0)
+							unread++;
+						if (flags & CAMEL_MESSAGE_DELETED)
+							deleted++;
+						if (flags & CAMEL_MESSAGE_JUNK) {
+							junked++;
+								if (! (flags & CAMEL_MESSAGE_DELETED))
+									junked_not_deleted++;						
+						}
+						if ((flags & (CAMEL_MESSAGE_DELETED|CAMEL_MESSAGE_JUNK)) == 0)
+							visible++;
+						camel_message_info_free(info);
+					}
+				}
+			}
+
+			switch (tag & CAMEL_ARG_TAG) {
+			case CAMEL_FOLDER_ARG_UNREAD:
+				if (vf->priv->unread_vfolder == 1)
+					count = unread == -1 ? 0 : unread - junked_not_deleted;
+				else
+					count = unread == -1 ? 0 : unread;
+				break;
+			case CAMEL_FOLDER_ARG_DELETED:
+				count = deleted == -1 ? 0 : deleted;
+				break;
+			case CAMEL_FOLDER_ARG_JUNKED:
+				count = junked == -1 ? 0 : junked;
+				break;
+			case CAMEL_FOLDER_ARG_JUNKED_NOT_DELETED:
+				count = junked_not_deleted == -1 ? 0 : junked_not_deleted;
+				break;				
+			case CAMEL_FOLDER_ARG_VISIBLE:
+				if (vf->priv->unread_vfolder == 1)
+					count = unread == -1 ? 0 : unread - junked_not_deleted;
+				else
+					count = visible == -1 ? 0 : visible;
+
+				break;
+			}
+			folder->summary->unread_count = unread == -1 ? 0 : unread;
+			folder->summary->deleted_count = deleted == -1 ? 0 : deleted;
+			junked = folder->summary->junk_count = junked == -1 ? 0 : junked;
+			folder->summary->junk_not_deleted_count = junked_not_deleted == -1 ? 0 : junked_not_deleted;
+			folder->summary->visible_count = visible == -1 ? 0 : visible;			
+			*arg->ca_int = count;
+			break;
+		default:
+			continue;
+		}
+
+		arg->tag = (tag & CAMEL_ARG_TYPE) | CAMEL_ARG_IGNORE;
+	}
+
+	return ((CamelObjectClass *)camel_vee_folder_parent)->getv(object, ex, args);
+}
+
 static void
 camel_vee_folder_class_init (CamelVeeFolderClass *klass)
 {
@@ -2105,6 +2079,8 @@ camel_vee_folder_class_init (CamelVeeFol
 
 	camel_vee_folder_parent = CAMEL_FOLDER_CLASS(camel_type_get_global_classfuncs (camel_folder_get_type ()));
 
+	((CamelObjectClass *)klass)->getv = vf_getv; 
+
 	folder_class->refresh_info = vee_refresh_info;
 	folder_class->sync = vee_sync;
 	folder_class->expunge = vee_expunge;
@@ -2152,11 +2128,13 @@ camel_vee_folder_init (CamelVeeFolder *o
 	obj->changes = camel_folder_change_info_new();
 	obj->search = camel_folder_search_new();
 	obj->hashes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
-	obj->loaded = g_hash_table_new (g_direct_hash, g_direct_equal);
+	/* Loaded is no longer used.*/
+	obj->loaded = NULL; 
 	obj->deleted = FALSE;
 	p->summary_lock = g_mutex_new();
 	p->subfolder_lock = g_mutex_new();
 	p->changed_lock = g_mutex_new();
+	p->unread_vfolder = -1;
 }
 
 void
@@ -2195,10 +2173,6 @@ vee_folder_stop_folder(CamelVeeFolder *v
 	camel_object_unhook_event((CamelObject *)sub, "folder_changed", (CamelObjectEventHookFunc) folder_changed, vf);
 	camel_object_unhook_event((CamelObject *)sub, "deleted", (CamelObjectEventHookFunc) subfolder_deleted, vf);
 	camel_object_unhook_event((CamelObject *)sub, "renamed", (CamelObjectEventHookFunc) folder_renamed, vf);
-	if (GPOINTER_TO_INT(g_hash_table_lookup (vf->loaded, sub))) {
-		g_hash_table_remove (vf->loaded, sub);
-		camel_object_unhook_event((CamelObject *)sub->summary, "summary_reloaded", (CamelObjectEventHookFunc) summary_reloaded, vf);
-	}
 	
 	p->folders = g_list_remove(p->folders, sub);
 	
@@ -2307,6 +2281,5 @@ camel_vee_folder_finalise (CamelObject *
 	g_mutex_free(p->subfolder_lock);
 	g_mutex_free(p->changed_lock);
 	g_hash_table_destroy (vf->hashes);
-	g_hash_table_destroy (vf->loaded);	
 	g_free(p);
 }
--- camel/camel-vee-summary.c	2009-02-04 09:13:36.000000000 +0530
+++ camel/camel-vee-summary.c	2009-02-04 10:42:04.000000000 +0530
@@ -171,24 +171,61 @@ vee_info_set_user_tag(CamelMessageInfo *
 	return res;
 }
 
-static gboolean
-vee_info_set_flags(CamelMessageInfo *mi, guint32 flags, guint32 set)
+void
+camel_vee_summary_load_check_unread_vfolder (CamelVeeSummary *vs)
 {
-	int res = FALSE;
-	CamelVeeFolder *vf = (CamelVeeFolder *)mi->summary->folder;
-	const char *exp = g_getenv("CAMEL_VFOLDER_UNREAD_EXP");
+	static int only_once = FALSE;
+	static char *exp = NULL;
+	char *meta;
 	gboolean hacked_unread_folder = FALSE;
+	CamelVeeFolder *vf = (CamelVeeFolder *) ((CamelFolderSummary *)vs)->folder;
 
 	/* HACK: Ugliest of all hacks. Its virtually not possible now
 	 * to maintain counts and the non matching uids of unread vfolder here.
 	 * So, I hardcode unread vfolder expression and hack it. */
+	if (!only_once) {
+		exp =  g_getenv("CAMEL_VFOLDER_UNREAD_EXP") ? g_strcompress(g_getenv("CAMEL_VFOLDER_UNREAD_EXP")) : NULL;
+		only_once = TRUE;
+	}
+	
 	if (!exp || !*exp)
-		exp = unread_str;
+		exp = g_strcompress(unread_str);
+
+	if (vf->expression && strstr(exp, vf->expression) &&  (vf->flags & CAMEL_STORE_VEE_FOLDER_SPECIAL) == 0)
+		hacked_unread_folder = TRUE;
+
+	meta = camel_object_meta_get (vf, "vfolder:unread");
+	if (!hacked_unread_folder && meta && strcmp (meta, "true") == 0)
+		hacked_unread_folder = TRUE;
+	g_free(meta);
+
+	if (hacked_unread_folder)
+		vf->priv->unread_vfolder = 1;
+	else
+		vf->priv->unread_vfolder = 0;
+}
+
+static gboolean
+vee_info_set_flags(CamelMessageInfo *mi, guint32 flags, guint32 set)
+{
+	int res = FALSE;
+	CamelVeeFolder *vf = (CamelVeeFolder *)mi->summary->folder;
+	gboolean hacked_unread_folder = FALSE;
+
 	if (camel_debug("vfolderexp"))
 		printf("Expression for vfolder '%s' is '%s'\n", mi->summary->folder->full_name, g_strescape(vf->expression, ""));
 
-	if (strstr(exp, vf->expression) &&  (vf->flags & CAMEL_STORE_VEE_FOLDER_SPECIAL) == 0)
-		hacked_unread_folder = TRUE;
+	if (vf->priv->unread_vfolder == -1)
+		camel_vee_summary_load_check_unread_vfolder (mi->summary);
+
+	if (vf->priv->unread_vfolder == 1)
+ 		hacked_unread_folder = TRUE;
+	else {
+		char *meta = camel_object_meta_get (mi->summary->folder, "vfolder:unread");
+		if (meta && strcmp (meta, "true") == 0)
+			hacked_unread_folder = TRUE;
+		g_free(meta);
+	}
 
 	if (mi->uid) {
 		guint32 old_visible, old_unread, old_deleted, old_junked, old_junked_not_deleted;
@@ -225,11 +262,11 @@ vee_info_set_flags(CamelMessageInfo *mi,
 			vsummary->fake_visible_count = mi->summary->visible_count;
 
 		/* Keep the summary in sync */
-		mi->summary->unread_count += unread - old_unread;
-		mi->summary->deleted_count += deleted - old_deleted;
-		mi->summary->junk_count += junked - old_junked;
-		mi->summary->junk_not_deleted_count += junked_not_deleted - old_junked_not_deleted;
-		mi->summary->visible_count += visible - old_visible;
+		//mi->summary->unread_count += unread - old_unread;
+		//mi->summary->deleted_count += deleted - old_deleted;
+		//mi->summary->junk_count += junked - old_junked;
+		//mi->summary->junk_not_deleted_count += junked_not_deleted - old_junked_not_deleted;
+		//mi->summary->visible_count += visible - old_visible;
 
 		if (vsummary->fake_visible_count || hacked_unread_folder)
 			vsummary->fake_visible_count += visible - old_visible;
@@ -418,6 +455,7 @@ camel_vee_summary_add(CamelVeeSummary *s
 	CamelVeeMessageInfo *mi;
 	char *vuid;
 
+	GHashTable * fcache;
 	vuid = g_malloc(strlen(uid)+9);
 	memcpy(vuid, hash, 8);
 	strcpy(vuid+8, uid);
@@ -440,6 +478,8 @@ camel_vee_summary_add(CamelVeeSummary *s
 
 	mi = (CamelVeeMessageInfo *)camel_message_info_new(&s->summary);
 	mi->summary = summary;
+	fcache = camel_folder_summary_get_flag_cache(summary);
+	mi->old_flags = GPOINTER_TO_UINT(g_hash_table_lookup (fcache, uid));
 	/* We would do lazy loading of flags, when the folders are loaded to memory through folder_reloaded signal */
 	camel_object_ref (summary);
 	mi->info.uid = (char *) camel_pstring_strdup (vuid);
--- camel/camel-vee-summary.h	2008-10-15 00:00:49.000000000 +0530
+++ camel/camel-vee-summary.h	2009-02-04 10:11:35.000000000 +0530
@@ -62,6 +62,7 @@ CamelFolderSummary *camel_vee_summary_ne
 
 CamelVeeMessageInfo * camel_vee_summary_add(CamelVeeSummary *s, CamelFolderSummary *summary, const char *uid, const char hash[8]);
 GPtrArray * camel_vee_summary_get_ids (CamelVeeSummary *summary, char hash[8]);
+void camel_vee_summary_load_check_unread_vfolder  (CamelVeeSummary *vs);
 
 G_END_DECLS
 
--- camel/camel-vtrash-folder.c	2008-11-07 10:04:23.000000000 +0530
+++ camel/camel-vtrash-folder.c	2009-02-04 10:11:35.000000000 +0530
@@ -151,7 +151,7 @@ vtrash_getv(CamelObject *object, CamelEx
 						guint32 flags;
 
 						vinfo = (CamelVeeMessageInfo *) info;
-						flags = vinfo->old_flags ? vinfo->old_flags : camel_message_info_flags(info);
+						flags = vinfo->old_flags;// ? vinfo->old_flags : camel_message_info_flags(info);
 
 						if ((flags & (CAMEL_MESSAGE_SEEN)) == 0)
 							unread++;
--- camel/providers/imap/camel-imap-folder.c	2009-02-04 09:13:36.000000000 +0530
+++ camel/providers/imap/camel-imap-folder.c	2009-02-04 10:11:35.000000000 +0530
@@ -964,6 +964,13 @@ imap_rescan (CamelFolder *folder, int ex
 			continue;
 
 		info = camel_folder_summary_uid (folder->summary, uid);
+		if (!info) {
+			if (g_getenv("CRASH_IMAP")) { /* Debug logs to tackle on hard to get imap crasher */
+				printf("CRASH: %s: %s", folder->full_name, uid);
+				g_assert(0);
+			} else
+				continue;
+		}
 
 		iinfo = (CamelImapMessageInfo *)info;
 
@@ -2941,9 +2948,26 @@ imap_get_message (CamelFolder *folder, c
 		 && retry < 2
 		 && camel_exception_get_id(ex) == CAMEL_EXCEPTION_SERVICE_UNAVAILABLE);
 
-done:	/* FIXME, this shouldn't be done this way. */
-	if (msg)
+done:
+	if (msg) {
+		/* FIXME, this shouldn't be done this way. */
 		camel_medium_set_header (CAMEL_MEDIUM (msg), "X-Evolution-Source", store->base_url);
+
+		if (!mi->info.mlist || !*mi->info.mlist) {
+			/* update mailing list information, if necessary */
+			char *mlist = camel_header_raw_check_mailing_list (&(CAMEL_MIME_PART (msg)->headers));
+
+			if (mlist) {
+				if (mi->info.mlist)
+					camel_pstring_free (mi->info.mlist);
+				mi->info.mlist = camel_pstring_add (mlist, TRUE);
+				mi->info.dirty = TRUE;
+
+				if (mi->info.summary)
+					camel_folder_summary_touch (mi->info.summary);
+			}
+		}
+	}
 fail:
 	camel_message_info_free(&mi->info);
 	
--- camel/providers/imap/camel-imap-journal.c	2008-10-13 13:37:30.000000000 +0530
+++ camel/providers/imap/camel-imap-journal.c	2009-02-04 10:11:35.000000000 +0530
@@ -160,6 +160,25 @@ free_uids (GPtrArray *array)
 }
 
 static GPtrArray *
+copy_uids_array (GPtrArray *array)
+{
+	GPtrArray *res;
+	guint i, sz;
+
+	if (!array)
+		return NULL;
+
+	sz = array->len;
+	res = g_ptr_array_sized_new (sz);
+
+	for (i = 0; i < sz; i++) {
+		g_ptr_array_add (res, g_strdup (g_ptr_array_index (array, i)));
+	}
+
+	return res;
+}
+
+static GPtrArray *
 decode_uids (FILE *file)
 {
 	GPtrArray *uids;
@@ -413,20 +432,20 @@ camel_imap_journal_log (CamelOfflineJour
 		{
 			GPtrArray *uids = va_arg (ap, GPtrArray *);
 			
-			entry->uids = uids;
+			entry->uids = copy_uids_array (uids);
 			break;
 		}
 		case CAMEL_IMAP_JOURNAL_ENTRY_APPEND:
 		{
 			char *uid = va_arg (ap, char *);
-			entry->append_uid = uid;
+			entry->append_uid = g_strdup (uid);
 			break;
 		}
 		case CAMEL_IMAP_JOURNAL_ENTRY_TRANSFER:
 		{
 			CamelFolder *dest = va_arg (ap, CamelFolder *);
 			
-			entry->uids = va_arg (ap, GPtrArray *);
+			entry->uids = copy_uids_array (va_arg (ap, GPtrArray *));
 			entry->move = va_arg (ap, gboolean);
 			entry->dest_folder_name = g_strdup (dest->full_name);
 			break;
--- camel/providers/imap/camel-imap-summary.c	2008-10-15 12:22:20.000000000 +0530
+++ camel/providers/imap/camel-imap-summary.c	2009-02-04 10:11:35.000000000 +0530
@@ -163,6 +163,22 @@ sort_uid_cmp (void *enc, int len1, void
 	return (a1 < a1) ? -1 : (a1 > a2) ? 1 : 0;
 }
 
+static int
+uid_compare (const void *va, const void *vb)
+{
+	const char **sa = (const char **)va, **sb = (const char **)vb;
+	unsigned long a, b;
+
+	a = strtoul (*sa, NULL, 10);
+	b = strtoul (*sb, NULL, 10);
+	if (a < b)
+		return -1;
+	else if (a == b)
+		return 0;
+	else
+		return 1;
+}
+
 /**
  * camel_imap_summary_new:
  * @folder: Parent folder.
@@ -181,7 +197,8 @@ camel_imap_summary_new (struct _CamelFol
 	camel_exception_init (&ex);
 
 	summary->folder = folder;
-	if (folder) {
+	/* Don't do DB sort. Its pretty slow to load */
+	if (folder && 0) {
 		camel_db_set_collate (folder->parent_store->cdb_r, "uid", "imap_uid_sort", (CamelDBCollate)sort_uid_cmp);
 		summary->sort_by = "uid";
 		summary->collate = "imap_uid_sort";
@@ -199,6 +216,8 @@ camel_imap_summary_new (struct _CamelFol
 		camel_exception_clear (&ex);
 	}
 
+	g_ptr_array_sort (summary->uids, (GCompareFunc) uid_compare); 
+
 	return summary;
 }
 
--- camel/providers/local/camel-local-private.h	2008-10-13 13:37:37.000000000 +0530
+++ camel/providers/local/camel-local-private.h	2009-02-04 10:11:35.000000000 +0530
@@ -42,6 +42,8 @@ struct _CamelLocalFolderPrivate {
 #define CAMEL_LOCAL_FOLDER_LOCK(f, l) (g_mutex_lock(((CamelLocalFolder *)f)->priv->l))
 #define CAMEL_LOCAL_FOLDER_UNLOCK(f, l) (g_mutex_unlock(((CamelLocalFolder *)f)->priv->l))
 
+int camel_local_frompos_sort (void *enc, int len1, void * data1, int len2, void *data2);
+
 G_END_DECLS
 
 #endif /* CAMEL_LOCAL_PRIVATE_H */
--- camel/providers/local/camel-local-summary.c	2008-10-14 11:00:40.000000000 +0530
+++ camel/providers/local/camel-local-summary.c	2009-02-04 10:11:35.000000000 +0530
@@ -484,6 +484,7 @@ local_summary_add(CamelLocalSummary *cls
 			
 			update_summary (s, (CamelMessageInfoBase *) mi, (CamelMessageInfoBase *) info);
 			mi->info.flags |= (camel_message_info_flags(info) & 0xffff);
+			camel_folder_summary_update_flag_cache (s, mi->info.uid, mi->info.flags);
 			mi->info.size = camel_message_info_size(info);
 		}
 
--- camel/providers/local/camel-mbox-summary.c	2008-10-14 11:00:40.000000000 +0530
+++ camel/providers/local/camel-mbox-summary.c	2009-02-04 10:11:35.000000000 +0530
@@ -46,6 +46,7 @@
 #include "camel-string-utils.h"
 #include "camel-store.h"
 #include "camel-folder.h"
+#include "camel-local-private.h"
 
 #define io(x)
 #define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/
@@ -211,29 +212,6 @@ camel_mbox_summary_finalise(CamelObject
 {
 	/*CamelMboxSummary *mbs = CAMEL_MBOX_SUMMARY(obj);*/
 }
-static int 
-frompos_sort (void *enc, int len1, void * data1, int len2, void *data2)
-{
-	static char *sa1=NULL, *sa2=NULL;
-	static int l1=0, l2=0;
-	int a1, a2;
-
-	if (l1 < len1+1) {
-		sa1 = g_realloc (sa1, len1+1);
-		l1 = len1+1;
-	}
-	if (l2 < len2+1) {
-		sa2 = g_realloc (sa2, len2+1);
-		l2 = len2+1;
-	}
-	strncpy (sa1, data1, len1);sa1[len1] = 0;
-	strncpy (sa2, data2, len2);sa2[len2] = 0;
-
-	a1 = strtoul (sa1, NULL, 10);
-	a2 = strtoul (sa2, NULL, 10);
-
-	return (a1 < a1) ? -1 : (a1 > a2) ? 1 : 0;
-}
 
 /**
  * camel_mbox_summary_new:
@@ -252,7 +230,7 @@ camel_mbox_summary_new(struct _CamelFold
 		CamelFolderSummary *summary = (CamelFolderSummary *)new;
 
 		/* Set the functions for db sorting */
-		camel_db_set_collate (folder->parent_store->cdb_r, "bdata", "mbox_frompos_sort", (CamelDBCollate)frompos_sort);
+		camel_db_set_collate (folder->parent_store->cdb_r, "bdata", "mbox_frompos_sort", (CamelDBCollate)camel_local_frompos_sort);
 		summary->sort_by = "bdata";
 		summary->collate = "mbox_frompos_sort";
 
--- camel/providers/local/camel-spool-summary.c	2008-10-14 11:00:40.000000000 +0530
+++ camel/providers/local/camel-spool-summary.c	2009-02-04 10:11:35.000000000 +0530
@@ -41,6 +41,7 @@
 #include "camel-store.h"
 
 #include "camel-spool-summary.h"
+#include "camel-local-private.h"
 
 #define io(x)
 #define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/
@@ -107,30 +108,6 @@ camel_spool_summary_finalise(CamelObject
 	/*CamelSpoolSummary *mbs = CAMEL_SPOOL_SUMMARY(obj);*/
 }
 
-static int 
-frompos_sort (void *enc, int len1, void * data1, int len2, void *data2)
-{
-	static char *sa1=NULL, *sa2=NULL;
-	static int l1=0, l2=0;
-	int a1, a2;
-
-	if (l1 < len1+1) {
-		sa1 = g_realloc (sa1, len1+1);
-		l1 = len1+1;
-	}
-	if (l2 < len2+1) {
-		sa2 = g_realloc (sa2, len2+1);
-		l2 = len2+1;
-	}
-	strncpy (sa1, data1, len1);sa1[len1] = 0;
-	strncpy (sa2, data2, len2);sa2[len2] = 0;
-
-	a1 = strtoul (sa1, NULL, 10);
-	a2 = strtoul (sa2, NULL, 10);
-
-	return (a1 < a1) ? -1 : (a1 > a2) ? 1 : 0;
-}
-
 CamelSpoolSummary *
 camel_spool_summary_new(struct _CamelFolder *folder, const char *mbox_name)
 {
@@ -138,7 +115,7 @@ camel_spool_summary_new(struct _CamelFol
 
 	((CamelFolderSummary *)new)->folder = folder;
 	if (folder) {
-		camel_db_set_collate (folder->parent_store->cdb_r, "bdata", "spool_frompos_sort", (CamelDBCollate)frompos_sort);
+		camel_db_set_collate (folder->parent_store->cdb_r, "bdata", "spool_frompos_sort", (CamelDBCollate)camel_local_frompos_sort);
 		((CamelFolderSummary *)new)->sort_by = "bdata";
 		((CamelFolderSummary *)new)->collate = "spool_frompos_sort";
 	}
--- camel/providers/local/Makefile.am	2008-10-13 13:37:37.000000000 +0530
+++ camel/providers/local/Makefile.am	2009-02-04 10:11:35.000000000 +0530
@@ -29,6 +29,7 @@ libcamellocal_la_SOURCES = 			\
 	camel-local-folder.c			\
 	camel-local-store.c			\
 	camel-local-summary.c			\
+	camel-local-private.c			\
 	camel-local-provider.c			\
 	camel-mbox-folder.c			\
 	camel-mbox-store.c			\
--- libedataserver/e-sexp.c	2008-10-13 13:38:18.000000000 +0530
+++ libedataserver/e-sexp.c	2009-02-04 10:11:35.000000000 +0530
@@ -706,7 +706,7 @@ e_sexp_term_eval(struct _ESExp *f, struc
 		r->value.time = t->value.time;
 		break;
 	case ESEXP_TERM_IFUNC:
-		if (t->value.func.sym->f.ifunc)
+		if (t->value.func.sym && t->value.func.sym->f.ifunc)
 			r = t->value.func.sym->f.ifunc(f, t->value.func.termcount, t->value.func.terms, t->value.func.sym->data);
 		break;
 	case ESEXP_TERM_FUNC:
--- /dev/null	2008-12-03 11:26:56.000000000 +0530
+++ camel/providers/local/camel-local-private.c	2009-01-29 08:42:09.000000000 +0530
@@ -0,0 +1,53 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; fill-column: 160 -*-
+ *
+ *  Copyright (C) 2008 Novell, Inc. (www.novell.com)
+ *
+ *  Authors: Srinivsa Ragavan <sragavan@novell.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU Lesser General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <glib.h>
+#include <string.h>
+#include <stdlib.h>
+#include "camel-local-private.h"
+
+int 
+camel_local_frompos_sort (void *enc, int len1, void * data1, int len2, void *data2)
+{
+	static char *sa1=NULL, *sa2=NULL;
+	static int l1=0, l2=0;
+	int a1, a2;
+
+	if (l1 < len1+1) {
+		sa1 = g_realloc (sa1, len1+1);
+		l1 = len1+1;
+	}
+	if (l2 < len2+1) {
+		sa2 = g_realloc (sa2, len2+1);
+		l2 = len2+1;
+	}
+	strncpy (sa1, data1, len1);sa1[len1] = 0;
+	strncpy (sa2, data2, len2);sa2[len2] = 0;
+
+	a1 = strtoul (sa1, NULL, 10);
+	a2 = strtoul (sa2, NULL, 10);
+
+	return a1 - a2;
+}

Places

File bnc-472719-eds-disk-summary-patches.patch of Package evolution-data-server

Places