Fix stale mailbox entries after deleted or moved

Refresh stale Maildir and sorted mailbox indexes when messages disappear, avoid rendering missing messages, and keep IMAP sort indexes in sync with mailbox count changes.

mailboxes/boxes-lib.pl

@@ -2087,10 +2087,20 @@ foreach $f (@files) {
 		$i++;
 		next;
 		}
+	local $idx = $i++;
 	local $mail = &read_mail_file($f, $_[3]);
-	$mail->{'idx'} = $i++;
-	$mail->{'id'} = $f;	# ID is relative path, like cur/4535534
-	$mail->{'id'} = substr($mail->{'id'}, length($_[0])+1);
+	if (!$mail && !$_[4]) {
+		# The cached Maildir file list can be stale if another client
+		# deleted or moved a message. Re-read it once before returning
+		# blank entries to the caller.
+		&flush_maildir_cachefile($_[0]);
+		return &list_maildir($_[0], $_[1], $_[2], $_[3], 1);
+		}
+	if ($mail) {
+		$mail->{'idx'} = $idx;
+		$mail->{'id'} = $f;	# ID is relative path, like cur/4535534
+		$mail->{'id'} = substr($mail->{'id'}, length($_[0])+1);
+		}
 	push(@rv, $mail);
 	}
 return @rv;
@@ -2110,9 +2120,11 @@ return map { substr($_, length($file)+1) } &get_maildir_files($file);
 sub select_maildir
 {
 local ($file, $ids, $headersonly) = @_;
+local $retried = $_[3];
 &mark_read_maildir($file);
 local @files = &get_maildir_files($file);
 local @rv;
+local $missing;
 foreach my $i (@$ids) {
 	local $path = "$file/$i";
 	local $mail = &read_mail_file($path, $headersonly);
@@ -2139,8 +2151,15 @@ foreach my $i (@$ids) {
 		# Get index in directory
 		$mail->{'idx'} = &indexof($path, @files);
 		}
+	else {
+		$missing = 1;
+		}
 	push(@rv, $mail);
 	}
+if ($missing && !$retried) {
+	&flush_maildir_cachefile($file);
+	return &select_maildir($file, $ids, $headersonly, 1);
+	}
 return @rv;
 }
 
@@ -2167,7 +2186,7 @@ else {
 	# Check the on-disk cache file
 	local $cachefile = &get_maildir_cachefile($_[0]);
 	local @cst = $cachefile ? stat($cachefile) : ( );
-	if ($cst[9] >= $newest) {
+	if ($cst[9] > $newest) {
 		# Can read the cache
 		open(CACHE, "<", $cachefile);
 		while(<CACHE>) {

mailboxes/folders-lib.pl

@@ -174,6 +174,7 @@ elsif ($_[2]->{'type'} == 4) {
 	local $count = $rv[2];
 	return () if (!$count);
 	$_[2]->{'lastchange'} = $rv[3] if ($rv[3]);
+	$_[2]->{'mailcount'} = $count;
 
 	# Work out what range we want
 	local ($start, $end) = &compute_start_end($_[0], $_[1], $count);
@@ -458,6 +459,7 @@ elsif ($folder->{'type'} == 4) {
 		}
 	local $h = $irv[1];
 	local $count = $irv[2];
+	$folder->{'mailcount'} = $count;
 	return () if (!$count);
         $folder->{'lastchange'} = $irv[3] if ($irv[3]);
 
@@ -637,8 +639,9 @@ elsif ($folder->{'type'} == 4) {
 		}
 	local $h = $rv[1];
 	local $count = $rv[2];
+	$folder->{'mailcount'} = $count;
 	return () if (!$count);
-        $folder->{'lastchange'} = $irv[3] if ($irv[3]);
+        $folder->{'lastchange'} = $rv[3] if ($rv[3]);
 
 	@rv = &imap_command($h, "FETCH 1:$count UID");
 	foreach my $uid (@{$rv[1]}) {
@@ -708,6 +711,8 @@ else {
 sub mailbox_list_mails_sorted
 {
 local ($start, $end, $folder, $headersonly, $error, $field, $dir) = @_;
+local ($requested_start, $requested_end) = ($start, $end);
+local $retried = $_[7];
 print DEBUG "mailbox_list_mails_sorted from $start to $end\n";
 if (!$field) {
 	# Default to current ordering
@@ -738,11 +743,25 @@ local @rv = map { undef } (0 .. scalar(@sorter)-1);
 local @wantids = map { $sorter[$_] } ($start .. $end);
 print DEBUG "wantids = ",scalar(@wantids),"\n";
 local @mails = &mailbox_select_mails($folder, \@wantids, $headersonly);
+local @missing;
 for(my $i=0; $i<@mails; $i++) {
+	if (!$mails[$i]) {
+		push(@missing, $wantids[$i]);
+		next;
+		}
 	$rv[$start+$i] = $mails[$i];
 	print DEBUG "setting $start+$i to ",$mails[$i]," id ",$wantids[$i],"\n";
 	$mails[$i]->{'sortidx'} = $start+$i;
 	}
+if (@missing && !$retried) {
+	# A sorted IMAP list can contain UIDs for messages that were
+	# expunged or moved by another client. Force one rebuild so stale
+	# entries don't render as blank 1969/no-subject rows.
+	&force_new_index_recheck($folder);
+	return &mailbox_list_mails_sorted($requested_start, $requested_end,
+					  $folder, $headersonly, $error,
+					  $field, $dir, 1);
+	}
 print DEBUG "rv = ",scalar(@rv),"\n";
 return @rv;
 }
@@ -808,7 +827,9 @@ local $ifile = &folder_new_sort_index_file($folder);
 &open_dbm_db($index, $ifile, 0600);
 print DEBUG "indexchange=$index->{'lastchange'} folderchange=$folder->{'lastchange'}\n";
 if ($index->{'lastchange'} != $folder->{'lastchange'} ||
-    !$folder->{'lastchange'}) {
+    !$folder->{'lastchange'} ||
+    (defined($folder->{'mailcount'}) &&
+     $index->{'mailcount'} != $folder->{'mailcount'})) {
 	# The mail file has changed .. get IDs and update the index with any
 	# that are missing
 	local @ids = &mailbox_idlist($folder);
@@ -823,6 +844,7 @@ if ($index->{'lastchange'} != $folder->{'lastchange'} ||
 	local @mails = scalar(@newids) ?
 			&mailbox_select_mails($folder, \@newids, 1) : ( );
 	foreach my $mail (@mails) {
+		next if (!$mail || !defined($mail->{'id'}));
 		foreach my $f (@index_fields) {
 			if ($f eq "date") {
 				# Convert date to Unix time

mailboxes/mailboxes-lib.pl

@@ -1209,6 +1209,7 @@ print &ui_columns_start(\@hcols, 100, 0, \@tds);
 # Show rows for actual mail messages
 my $i = 0;
 foreach my $mail (@mail) {
+	next if (!$mail);
 	local $idx = $mail->{'idx'};
 	local $cols = 0;
 	local @cols;