Merge pull request #2744 from webmin/dev/fix-stale-mb

Fix stale mailbox entries after deleted or moved

mailboxes/boxes-lib.pl

@@ -2087,10 +2087,20 @@ foreach $f (@files) {
 		$i++;
 		next;
 		}
+	local $idx = $i++;
 	local $mail = &read_mail_file($f, $_[3]);
-	$mail->{'idx'} = $i++;
-	$mail->{'id'} = $f;	# ID is relative path, like cur/4535534
-	$mail->{'id'} = substr($mail->{'id'}, length($_[0])+1);
+	if (!$mail && !$_[4]) {
+		# The cached Maildir file list can be stale if another client
+		# deleted or moved a message. Re-read it once before returning
+		# blank entries to the caller.
+		&flush_maildir_cachefile($_[0]);
+		return &list_maildir($_[0], $_[1], $_[2], $_[3], 1);
+		}
+	if ($mail) {
+		$mail->{'idx'} = $idx;
+		$mail->{'id'} = $f;	# ID is relative path, like cur/4535534
+		$mail->{'id'} = substr($mail->{'id'}, length($_[0])+1);
+		}
 	push(@rv, $mail);
 	}
 return @rv;
@@ -2110,9 +2120,11 @@ return map { substr($_, length($file)+1) } &get_maildir_files($file);
 sub select_maildir
 {
 local ($file, $ids, $headersonly) = @_;
+local $retried = $_[3];
 &mark_read_maildir($file);
 local @files = &get_maildir_files($file);
 local @rv;
+local $missing;
 foreach my $i (@$ids) {
 	local $path = "$file/$i";
 	local $mail = &read_mail_file($path, $headersonly);
@@ -2139,8 +2151,15 @@ foreach my $i (@$ids) {
 		# Get index in directory
 		$mail->{'idx'} = &indexof($path, @files);
 		}
+	else {
+		$missing = 1;
+		}
 	push(@rv, $mail);
 	}
+if ($missing && !$retried) {
+	&flush_maildir_cachefile($file);
+	return &select_maildir($file, $ids, $headersonly, 1);
+	}
 return @rv;
 }
 
@@ -2167,7 +2186,7 @@ else {
 	# Check the on-disk cache file
 	local $cachefile = &get_maildir_cachefile($_[0]);
 	local @cst = $cachefile ? stat($cachefile) : ( );
-	if ($cst[9] >= $newest) {
+	if ($cst[9] > $newest) {
 		# Can read the cache
 		open(CACHE, "<", $cachefile);
 		while(<CACHE>) {
@@ -2188,6 +2207,32 @@ else {
 					# Flagged as deleted by IMAP .. skip
 					next;
 					}
+				# Skip entries that cannot be read as messages.
+				local $path = "$_[0]/$d/$f";
+				local @fst = stat($path);
+				if (!@fst) {
+					&error_stderr("Skipping Maildir file ".
+						      "$path : stat failed : ".
+						      "$!");
+					next;
+					}
+				if (!$fst[7]) {
+					&error_stderr("Skipping Maildir file ".
+						      "$path : file is zero ".
+						      "bytes");
+					next;
+					}
+				if (!-r _) {
+					my $m = sprintf("%04o",$fst[2] & 07777);
+					my $o = getpwuid($fst[4]) || $fst[4];
+					my $g = getgrgid($fst[5]) || $fst[5];
+					&error_stderr("Skipping Maildir file ".
+						      "$path : not readable by".
+						      " current user, owner=".
+						      "$o($fst[4]):$g($fst[5])".
+						      " mode=$m");
+					next;
+					}
 				push(@shorts, "$d/$f")
 				}
 			closedir(DIR);
@@ -2661,11 +2706,13 @@ sub read_mail_file
 my ($file, $headersonly) = @_;
 
 # Open and read the mail file
+local @st = stat($file);
+return undef if (@st && !$st[7]);
 &open_as_mail_user(MAIL, $file) || return undef;
 my $mail = &read_mail_fh(MAIL, 0, $headersonly);
-$mail->{'file'} = $file;
 close(MAIL);
-local @st = stat($file);
+
+$mail->{'file'} = $file;
 $mail->{'size'} = $st[7];
 $mail->{'time'} = $st[9];
 $mail->{'ctime'} = $st[10];

mailboxes/folders-lib.pl

@@ -174,6 +174,7 @@ elsif ($_[2]->{'type'} == 4) {
 	local $count = $rv[2];
 	return () if (!$count);
 	$_[2]->{'lastchange'} = $rv[3] if ($rv[3]);
+	$_[2]->{'mailcount'} = $count;
 
 	# Work out what range we want
 	local ($start, $end) = &compute_start_end($_[0], $_[1], $count);
@@ -458,6 +459,7 @@ elsif ($folder->{'type'} == 4) {
 		}
 	local $h = $irv[1];
 	local $count = $irv[2];
+	$folder->{'mailcount'} = $count;
 	return () if (!$count);
         $folder->{'lastchange'} = $irv[3] if ($irv[3]);
 
@@ -637,8 +639,9 @@ elsif ($folder->{'type'} == 4) {
 		}
 	local $h = $rv[1];
 	local $count = $rv[2];
+	$folder->{'mailcount'} = $count;
 	return () if (!$count);
-        $folder->{'lastchange'} = $irv[3] if ($irv[3]);
+        $folder->{'lastchange'} = $rv[3] if ($rv[3]);
 
 	@rv = &imap_command($h, "FETCH 1:$count UID");
 	foreach my $uid (@{$rv[1]}) {
@@ -708,6 +711,8 @@ else {
 sub mailbox_list_mails_sorted
 {
 local ($start, $end, $folder, $headersonly, $error, $field, $dir) = @_;
+local ($requested_start, $requested_end) = ($start, $end);
+local $retried = $_[7];
 print DEBUG "mailbox_list_mails_sorted from $start to $end\n";
 if (!$field) {
 	# Default to current ordering
@@ -738,11 +743,25 @@ local @rv = map { undef } (0 .. scalar(@sorter)-1);
 local @wantids = map { $sorter[$_] } ($start .. $end);
 print DEBUG "wantids = ",scalar(@wantids),"\n";
 local @mails = &mailbox_select_mails($folder, \@wantids, $headersonly);
+local @missing;
 for(my $i=0; $i<@mails; $i++) {
+	if (!$mails[$i]) {
+		push(@missing, $wantids[$i]);
+		next;
+		}
 	$rv[$start+$i] = $mails[$i];
 	print DEBUG "setting $start+$i to ",$mails[$i]," id ",$wantids[$i],"\n";
 	$mails[$i]->{'sortidx'} = $start+$i;
 	}
+if (@missing && !$retried) {
+	# A sorted IMAP list can contain UIDs for messages that were
+	# expunged or moved by another client. Force one rebuild so stale
+	# entries don't render as blank 1969/no-subject rows.
+	&force_new_index_recheck($folder);
+	return &mailbox_list_mails_sorted($requested_start, $requested_end,
+					  $folder, $headersonly, $error,
+					  $field, $dir, 1);
+	}
 print DEBUG "rv = ",scalar(@rv),"\n";
 return @rv;
 }
@@ -808,7 +827,9 @@ local $ifile = &folder_new_sort_index_file($folder);
 &open_dbm_db($index, $ifile, 0600);
 print DEBUG "indexchange=$index->{'lastchange'} folderchange=$folder->{'lastchange'}\n";
 if ($index->{'lastchange'} != $folder->{'lastchange'} ||
-    !$folder->{'lastchange'}) {
+    !$folder->{'lastchange'} ||
+    (defined($folder->{'mailcount'}) &&
+     $index->{'mailcount'} != $folder->{'mailcount'})) {
 	# The mail file has changed .. get IDs and update the index with any
 	# that are missing
 	local @ids = &mailbox_idlist($folder);
@@ -823,6 +844,7 @@ if ($index->{'lastchange'} != $folder->{'lastchange'} ||
 	local @mails = scalar(@newids) ?
 			&mailbox_select_mails($folder, \@newids, 1) : ( );
 	foreach my $mail (@mails) {
+		next if (!$mail || !defined($mail->{'id'}));
 		foreach my $f (@index_fields) {
 			if ($f eq "date") {
 				# Convert date to Unix time

mailboxes/mailboxes-lib.pl

@@ -1209,6 +1209,7 @@ print &ui_columns_start(\@hcols, 100, 0, \@tds);
 # Show rows for actual mail messages
 my $i = 0;
 foreach my $mail (@mail) {
+	next if (!$mail);
 	local $idx = $mail->{'idx'};
 	local $cols = 0;
 	local @cols;