Don't read every mail file on every page load

mailboxes/boxes-lib.pl

@@ -175,7 +175,7 @@ local ($file, $ids, $headersonly) = @_;
 local @rv;
 
 local (@rv);
-my %index;
+my (%index, %midmap);
 local $gotindex;
 
 local $umf = &user_mail_file($file);
@@ -191,24 +191,23 @@ foreach my $i (@$ids) {
 			($1 eq '-' && !$dash) ? 0 : 1;
 	print DEBUG "seeking to $pos in $umf, got $ll";
 	if (!$fromok) {
-		# Oh noes! Need to find it
+		# Oh noes! Need to find it, using a map from message IDs to
+		# positions built once from the index
 		if (!$gotindex++) {
 			&build_dbm_index($file, \%index);
-			}
-		$pos = undef;
-		while(my ($k, $v) = each %index) {
-			if (int($k) eq $k) {
-				my ($p, $line, $subject, $from, $mid)=
-					split(/\0/, $v);
-				if ($mid eq $wantmid) {
-					# Found it!
-					$pos = $p;
-					$idx = $k;
-					$startline = $line;
-					last;
+			while(my ($k, $v) = each %index) {
+				if (int($k) eq $k) {
+					my ($p, $line, $subject, $from, $mid)=
+						split(/\0/, $v);
+					$midmap{$mid} ||= [ $p, $k, $line ];
 					}
 				}
 			}
+		$pos = undef;
+		if ($midmap{$wantmid}) {
+			# Found it!
+			($pos, $idx, $startline) = @{$midmap{$wantmid}};
+			}
 		}
 
 	if (defined($pos)) {
@@ -2128,6 +2127,17 @@ local ($file, $ids, $headersonly) = @_;
 local $retried = $_[3];
 &mark_read_maildir($file);
 local @files = &get_maildir_files($file);
+
+# Build maps from paths to directory indexes, and from paths without flags to
+# current paths, so that each lookup doesn't need to scan the whole list
+my (%fileidx, %basefile);
+for(my $i=0; $i<@files; $i++) {
+	$fileidx{$files[$i]} = $i;
+	my $base = $files[$i];
+	$base =~ s/:2,[A-Za-z]*$//;
+	$basefile{$base} ||= $files[$i];
+	}
+
 local @rv;
 local $missing;
 foreach my $i (@$ids) {
@@ -2135,10 +2145,7 @@ foreach my $i (@$ids) {
 	local $mail = &read_mail_file($path, $headersonly);
 	if (!$mail && $path =~ /^(.*)\/(cur|tmp|new)\/([^:]*)(:2,([A-Za-z]*))?$/) {
 		# Flag may have changed - update path
-		local $suffix = "$2/$3";
-		local ($newfile) = grep
-		  { substr($_, length($file)+1, length($suffix)) eq $suffix }
-		  @files;
+		local $newfile = $basefile{"$1/$2/$3"};
 		if ($newfile) {
 			$path = $newfile;
 			$mail = &read_mail_file($path, $headersonly);
@@ -2154,7 +2161,7 @@ foreach my $i (@$ids) {
 		$mail->{'id'} = $path;
 		$mail->{'id'} = substr($mail->{'id'}, length($file)+1);
 		# Get index in directory
-		$mail->{'idx'} = &indexof($path, @files);
+		$mail->{'idx'} = defined($fileidx{$path}) ? $fileidx{$path} : -1;
 		}
 	else {
 		$missing = 1;
@@ -2199,7 +2206,6 @@ else {
 			push(@files, $_[0]."/".$_);
 			}
 		close(CACHE);
-		$main::list_maildir_cache_time{$_[0]} = $cst[9];
 		}
 	else {
 		# Really read
@@ -2261,9 +2267,9 @@ else {
 							   undef, $cachefile);
 				}
 			}
-		$main::list_maildir_cache_time{$_[0]} = $st[9];
 		}
 	$main::list_maildir_cache{$_[0]} = \@files;
+	$main::list_maildir_cache_time{$_[0]} = $newest;
 	}
 return @files;
 }
@@ -2301,20 +2307,38 @@ sub mark_read_maildir
 local ($dir) = @_;
 local @files = &get_maildir_files($dir);
 local $i = 0;
+local $changed = 0;
+local $stale = 0;
 foreach my $nf (@files) {
 	if (substr($nf, length($dir)+1, 3) eq "new") {
 		local $cf = $nf;
-		$cf =~ s/\/new\//\/cur\//g;
-		if (&rename_as_mail_user($nf, $cf)) {
+		$cf =~ s/\/new\/([^\/]+)$/\/cur\/$1/;
+		if (!-e $nf) {
+			# Already moved or deleted by another process, so the
+			# cached file list is out of date. Don't try to rename,
+			# as rename_file would fall back to running mv
+			$stale = 1;
+			}
+		elsif (&rename_as_mail_user($nf, $cf)) {
 			$files[$i] = $cf;
 			$changed = 1;
 			}
 		}
 	$i++;
 	}
-if ($changed) {
-	# Update the cache
+if ($stale) {
+	# Force a re-read of the directory next time
+	&flush_maildir_cachefile($dir);
+	}
+elsif ($changed) {
+	# Update the cache, including the new directory modification time
+	local $newest;
+	foreach my $d ("$dir/cur", "$dir/new") {
+		local @dst = stat($d);
+		$newest = $dst[9] if ($dst[9] > $newest);
+		}
 	$main::list_maildir_cache{$dir} = \@files;
+	$main::list_maildir_cache_time{$dir} = $newest;
 	local $cachefile = &get_maildir_cachefile($dir);
 	if ($cachefile) {
 		&open_tempfile(CACHE, ">$cachefile", 1);

mailboxes/folders-lib.pl

@@ -572,20 +572,22 @@ foreach my $mail (@mail) {
 return @mail;
 }
 
-# mailbox_get_mail(&folder, id, headersonly)
-# Convenience function to get a single mail by ID
+# mailbox_get_mail(&folder, id, headersonly, [skip-sort-index])
+# Convenience function to get a single mail by ID. Unless skip-sort-index is
+# set, the message's position in the sorted folder is also computed, which
+# can be slow for large folders.
 sub mailbox_get_mail
 {
-local ($folder, $id, $headersonly) = @_;
+local ($folder, $id, $headersonly, $nosortidx) = @_;
 local ($mail) = &mailbox_select_mails($folder, [ $id ], $headersonly);
-if ($mail) {
+if ($mail && !$nosortidx) {
 	# Find the sort index for this message
 	local ($field, $dir) = &get_sort_field($folder);
 	if (!$field || !$folder->{'sortable'}) {
 		# No sorting, so sort index is the opposite of real
 		$mail->{'sortidx'} = &mailbox_folder_size($folder, 1) -
 				     $mail->{'idx'} - 1;
-		print DEBUG "idx=$mail->{'idx'} sortidx=$mail->{'sortidx'} size=",&mailbox_folder_size($folder, 1),"\n";
+		print DEBUG "idx=$mail->{'idx'} sortidx=$mail->{'sortidx'}\n";
 		}
 	else {
 		# Need to extract from sort index
@@ -643,10 +645,20 @@ elsif ($folder->{'type'} == 4) {
 	return () if (!$count);
         $folder->{'lastchange'} = $rv[3] if ($rv[3]);
 
-	@rv = &imap_command($h, "FETCH 1:$count UID");
-	foreach my $uid (@{$rv[1]}) {
-		if ($uid =~ /UID\s+(\d+)/) {
-			push(@idlist, $1);
+	# UID SEARCH returns all UIDs on one line, which is much faster to
+	# read than one FETCH response per message
+	@rv = &imap_command($h, "UID SEARCH ALL");
+	if ($rv[0]) {
+		local ($srch) = grep { $_ =~ /^\*\s+SEARCH/i } @{$rv[1]};
+		$srch =~ s/^\*\s+SEARCH\s*//i;
+		@idlist = sort { $a <=> $b } grep { /^\d+$/ } split(/\s+/, $srch);
+		}
+	else {
+		@rv = &imap_command($h, "FETCH 1:$count UID");
+		foreach my $uid (@{$rv[1]}) {
+			if ($uid =~ /UID\s+(\d+)/) {
+				push(@idlist, $1);
+				}
 			}
 		}
 	}
@@ -1853,7 +1865,7 @@ elsif ($f->{'type'} == 6 && !$est) {
 	# exist in the parent folders
 	$rv = 0;
 	foreach my $msg (@{$f->{'members'}}) {
-		if (&mailbox_get_mail($msg->[0], $msg->[1])) {
+		if (&mailbox_get_mail($msg->[0], $msg->[1], 1, 1)) {
 			$rv++;
 			}
 		}
@@ -1904,12 +1916,19 @@ elsif ($folder->{'type'} == 5) {
 	}
 else {
 	# For all other folders, just check individual messages
-	# XXX faster for maildir?
 	local @data = ( 0, 0, 0 );
 	local @mails;
 	eval {
 		$main::error_must_die = 1;
-		@mails = &mailbox_list_mails(undef, undef, $folder, 1);
+		if ($folder->{'type'} == 1 && $folder->{'flags'}) {
+			# For a Maildir where read flags come from filenames,
+			# only read the headers of messages with no flags set,
+			# as only they need the read DBM checked
+			@mails = &list_maildir_flags($folder);
+			}
+		else {
+			@mails = &mailbox_list_mails(undef, undef, $folder, 1);
+			}
 		};
 	return ( ) if ($@);
 	foreach my $m (@mails) {
@@ -1926,6 +1945,36 @@ else {
 	}
 }
 
+# list_maildir_flags(&folder)
+# Returns messages in a Maildir folder with IDs and flags taken from their
+# filenames. Headers are only read for messages with no flags.
+sub list_maildir_flags
+{
+my ($folder) = @_;
+my $md = $folder->{'file'};
+my @rv;
+&switch_to_folder_user($folder);
+&mark_read_maildir($md);
+foreach my $f (&get_maildir_files($md)) {
+	my $mail = { 'header' => { } };
+	if ($f =~ /:2,([A-Za-z]*)$/) {
+		my @flags = split(//, $1);
+		$mail->{'read'} = &indexoflc("S", @flags) >= 0 ? 1 : 0;
+		$mail->{'special'} = &indexoflc("F", @flags) >= 0 ? 1 : 0;
+		$mail->{'replied'} = &indexoflc("R", @flags) >= 0 ? 1 : 0;
+		$mail->{'flags'} = 1;
+		}
+	if (!$mail->{'read'} && !$mail->{'special'} && !$mail->{'replied'}) {
+		$mail = &read_mail_file($f, 1) || next;
+		}
+	$mail->{'file'} = $f;
+	$mail->{'id'} = substr($f, length($md)+1);
+	push(@rv, $mail);
+	}
+&switch_from_folder_user($folder);
+return @rv;
+}
+
 # mailbox_set_read_flags(&folder, &mail, read, special, replied)
 # Updates the status flags on some message
 sub mailbox_set_read_flag