view_mail.cgi performance fixes

Fix CGI response status handling

acl/edit_user.cgi

@@ -90,18 +90,25 @@ my @mcan = $access{'gassign'} eq '*' ?
 		( ( map { $_->{'name'} } @glist ), '_none' ) :
 		split(/\s+/, $access{'gassign'});
 my %gcan = map { $_, 1 } @mcan;
+# Find the user's group even when this editor cannot change it.
 my $memg;
+foreach my $g (@glist) {
+	if (&indexof($user{'name'}, @{$g->{'members'}}) >= 0 ||
+	    $in{'clone'} &&
+	     &indexof($in{'clone'}, @{$g->{'members'}}) >= 0) {
+		# A clone starts in the same group as its source user.
+		$memg = $g;
+		}
+	}
+# Show the group selector when this editor may change membership.
 if (@glist && %gcan) {
 	my @opts = ( );
 	if ($gcan{'_none'}) {
+		# Allow leaving a group only when the editor has permission.
 		push(@opts, [ undef, "<$text{'edit_none'}>" ]);
 		}
+	# Keep the current group available even if it cannot be newly assigned.
 	foreach my $g (@glist) {
-		if (&indexof($user{'name'}, @{$g->{'members'}}) >= 0 ||
-		    $in{'clone'} &&
-		     &indexof($in{'clone'}, @{$g->{'members'}}) >= 0) {
-			$memg = $g;
-			}
 		next if (!$gcan{$g->{'name'}} && $memg ne $g);
 		push(@opts, [ $g->{'name'} ]);
 		}
@@ -375,8 +382,8 @@ elsif ($miniserv{'twofactor_provider'}) {
 		&ui_submit($text{'edit_twofactoradd'}, "twofactor"));
 	}
 
-# Can accept RPC calls?
-if ($access{'acl'} && !$safe) {
+# RPC is a global permission and cannot be edited on a group member.
+if ($access{'acl'} && !$memg->{'name'} && !$safe) {
 	print &ui_table_row(&hlink($text{'acl_rpc'}, 'rpc'),
 		&ui_radio("rpc", int($uaccess{'rpc'}),
 			  [ [ 1, $text{'acl_rpc1'} ],
@@ -462,9 +469,9 @@ print &ui_table_row(undef, &ui_links_row(\@links).
                            &ui_links_row(\@links), 2);
 print &ui_hidden_table_end("mods");
 
-# Add global ACL section, but only if not set from the group
-my $groupglobal = $memg && -r "$config_directory/$memg->{'name'}.acl";
-if ($access{'acl'} && !$groupglobal && !$safe) {
+# Group members inherit global permissions, even without a local ACL file.
+# Hide these fields because save_user.cgi ignores individual edits.
+if ($access{'acl'} && !$memg->{'name'} && !$safe) {
 	print &ui_hidden_table_start($text{'edit_global'}, "width=100%", 2,
 				     "global", 0, [ "width=30%" ]);
 	print &ui_hidden("acl_security_form", 1);

gray-theme/unauthenticated/gray-theme.css

@@ -23,12 +23,10 @@
 	--radius: 8px;
 	--radius-sm: 6px;
 	--control-height: 32px;
-	/* Room between cards, buttons and blocks, and the same room around a
-	 * page at the top and the sides; the bottom keeps scroll room under
-	 * the last element */
-	--card-gap: 10px;
-	--page-pad-x: var(--card-gap);
-	--page-pad-top: var(--card-gap);
+	/* Shared gap around the menu and between cards, buttons and blocks.
+	 * Light surfaces use a slightly tighter gap than dark surfaces.
+	 * The page bottom keeps scroll room under the last element. */
+	--card-gap: 4px;
 	--page-pad-bottom: 28px;
 	/* Titles on the ground sit a little in from the card edge, so they
 	 * neither touch the edge nor go as deep as the text inside the
@@ -82,6 +80,8 @@
 	--accent-color-hover: #234d93;
 	--accent-color-soft: #a9c0e8;
 	--element-highlight-bg-color: #e3ebf8;
+	/* Navigation loader, with a separate shade for each color scheme. */
+	--menu-loader-color: #dc143c;
 	/* Text on a button filled with a notice color: white on the deep
 	 * light-mode colors, the ground color on the bright dark-mode ones.
 	 * The dark palette keeps its own amber for the warning button. */
@@ -126,6 +126,7 @@
 @media (prefers-color-scheme: dark) {
 	html:not([data-scheme="light"]) body:not([data-scheme="light"]) {
 		color-scheme: dark;
+		--card-gap: 7px;
 		--shadow: 0 1px 2px rgba(0, 0, 0, 0.3);
 		--shadow-color: rgba(0, 0, 0, 0.4);
 		--border-color-panel: #30353d;
@@ -159,6 +160,7 @@
 		--accent-color-hover: #5a8ad8;
 		--accent-color-soft: #3d5f8a;
 		--element-highlight-bg-color: #26395a;
+		--menu-loader-color: #c7302c;
 		--alert-button-text: var(--bg-color);
 		--alert-button-warning: #e0a63c;
 		--alert-button-warning-text: var(--bg-color);
@@ -202,6 +204,7 @@
 }
 html[data-scheme="dark"] body, body[data-scheme="dark"] {
 	color-scheme: dark;
+	--card-gap: 7px;
 	--shadow: 0 1px 2px rgba(0, 0, 0, 0.3);
 	--shadow-color: rgba(0, 0, 0, 0.4);
 	--border-color-panel: #30353d;
@@ -235,6 +238,7 @@ html[data-scheme="dark"] body, body[data-scheme="dark"] {
 	--accent-color-hover: #5a8ad8;
 	--accent-color-soft: #3d5f8a;
 	--element-highlight-bg-color: #26395a;
+	--menu-loader-color: #c7302c;
 	--alert-button-text: var(--bg-color);
 	--alert-button-warning: #e0a63c;
 	--alert-button-warning-text: var(--bg-color);
@@ -293,6 +297,9 @@ html {
 	background: transparent;
 }
 body {
+	/* Resolve page insets here so they follow the body's color scheme. */
+	--page-pad-x: var(--card-gap);
+	--page-pad-top: var(--card-gap);
 	margin: 0;
 	padding: var(--page-pad-top) var(--page-pad-x) var(--page-pad-bottom);
 	/* The core header sets height:100% inline on html and body, which
@@ -1690,16 +1697,16 @@ td:not(:hover) .ui_link_help { visibility: hidden; }
 .ui_text_wrap { float: right; }
 /* Tooltips from the aria-label of elements marked data-tooltip: a dark
  * box with an arrow above the element, shown on hover */
-[data-tooltip][aria-label] {
+[data-tooltip][aria-label]:not(.ui_tip) {
 	position: relative;
 	vertical-align: top;
 }
 /* Card actions align help bubbles through ui-lib, but this theme still
  * needs the bubble to anchor its generated tooltip. */
-.ui_card_actions .ui_help[data-tooltip][aria-label] {
+.ui_card_actions .ui_help[data-tooltip][aria-label]:not(.ui_tip) {
 	position: relative;
 }
-[data-tooltip][aria-label]::after {
+[data-tooltip][aria-label]:not(.ui_tip)::after {
 	content: "";
 	position: absolute;
 	bottom: 100%;
@@ -1717,7 +1724,7 @@ td:not(:hover) .ui_link_help { visibility: hidden; }
 	/* Keep a tooltip near the top above the sticky page header. */
 	z-index: 10;
 }
-[data-tooltip][aria-label]::before {
+[data-tooltip][aria-label]:not(.ui_tip)::before {
 	content: attr(aria-label);
 	position: absolute;
 	bottom: calc(15px + 100%);
@@ -1740,8 +1747,8 @@ td:not(:hover) .ui_link_help { visibility: hidden; }
 	visibility: hidden;
 	z-index: 10;
 }
-[data-tooltip][aria-label]:hover::before,
-[data-tooltip][aria-label]:hover::after {
+[data-tooltip][aria-label]:not(.ui_tip):hover::before,
+[data-tooltip][aria-label]:not(.ui_tip):hover::after {
 	visibility: visible;
 	opacity: 1;
 	transition-duration: 0.5s;
@@ -1749,7 +1756,7 @@ td:not(:hover) .ui_link_help { visibility: hidden; }
 /* Opacity gives the tooltip trigger its own stacking context. Raise that
  * context while the tooltip is visible, because raising its children alone
  * cannot place them above the sticky header. */
-[data-tooltip][aria-label]:hover {
+[data-tooltip][aria-label]:not(.ui_tip):hover {
 	z-index: 10;
 }
 
@@ -1812,7 +1819,7 @@ br[data-x-br] {
 /* ---- Left frame (left.cgi) ---- */
 
 /* The frame shows the page ground, and the menu floats on it as a card
- * with a 7px gap at the top, the left and the bottom, and rounder
+ * with the shared gap at the top, the left and the bottom, and rounder
  * corners than the other cards. At the right the card meets the frame
  * edge, so the page padding of the right frame is the whole gap between
  * the card and the content, and a page opened in a window of its own
@@ -1822,7 +1829,7 @@ br[data-x-br] {
 body.left-frame {
 	box-sizing: border-box;
 	height: 100vh !important;
-	padding: 7px 0 7px 7px;
+	padding: var(--card-gap) 0 var(--card-gap) var(--card-gap);
 	overflow: hidden;
 	background: var(--bg-color);
 }
@@ -1867,11 +1874,9 @@ html.loading-right .menu-loader { display: block; }
 	width: 35%;
 	height: 100%;
 	border-radius: 2px;
-	background: var(--accent-color);
+	background: var(--menu-loader-color);
 	animation: menu-bounce 0.9s ease-in-out infinite alternate;
 }
-/* Match Authentic's crimson progress bar for Cloudmin navigation. */
-.menu-panel-cloudmin .menu-loader::before { background: #dc143c; }
 @keyframes menu-bounce { to { left: 65%; } }
 /* The brand block: the logo has as much room above it as below it, and
  * the logo files are trimmed to their drawing, so every product sits the

htaccess-htpasswd/search.cgi

@@ -19,7 +19,8 @@ $in{'search'} =~ /^\// && $in{'search'} !~ /\.\./ ||
 
 @dirs = &list_directories();
 %got = map { ( "$_->[0]/$config{'htaccess'}", 1 ) } @dirs;
-print "<b>",&text('search_doing', "<tt>$in{'search'}</tt>"),"</b><p>\n";
+print "<b>",&text('search_doing',
+		  "<tt>".&html_escape($in{'search'})."</tt>"),"</b><p>\n";
 
 # Use the find command
 &switch_user();

init/edit_rc.cgi

@@ -36,7 +36,8 @@ if ($in{'new'}) {
 	}
 else {
 	# Just show fill action file contents
-	print &ui_table_row($text{'edit_name'}, "<tt>$in{'name'}</tt>");
+	print &ui_table_row($text{'edit_name'},
+			    "<tt>".&html_escape($in{'name'})."</tt>");
 	print &ui_hidden("name", $in{'name'});
 
 	$script = &read_file_contents($rc->{'file'});

mailboxes/boxes-lib.pl

@@ -175,7 +175,7 @@ local ($file, $ids, $headersonly) = @_;
 local @rv;
 
 local (@rv);
-my %index;
+my (%index, %midmap);
 local $gotindex;
 
 local $umf = &user_mail_file($file);
@@ -191,24 +191,23 @@ foreach my $i (@$ids) {
 			($1 eq '-' && !$dash) ? 0 : 1;
 	print DEBUG "seeking to $pos in $umf, got $ll";
 	if (!$fromok) {
-		# Oh noes! Need to find it
+		# Oh noes! Need to find it, using a map from message IDs to
+		# positions built once from the index
 		if (!$gotindex++) {
 			&build_dbm_index($file, \%index);
-			}
-		$pos = undef;
-		while(my ($k, $v) = each %index) {
-			if (int($k) eq $k) {
-				my ($p, $line, $subject, $from, $mid)=
-					split(/\0/, $v);
-				if ($mid eq $wantmid) {
-					# Found it!
-					$pos = $p;
-					$idx = $k;
-					$startline = $line;
-					last;
+			while(my ($k, $v) = each %index) {
+				if (int($k) eq $k) {
+					my ($p, $line, $subject, $from, $mid)=
+						split(/\0/, $v);
+					$midmap{$mid} ||= [ $p, $k, $line ];
 					}
 				}
 			}
+		$pos = undef;
+		if ($midmap{$wantmid}) {
+			# Found it!
+			($pos, $idx, $startline) = @{$midmap{$wantmid}};
+			}
 		}
 
 	if (defined($pos)) {
@@ -2128,6 +2127,8 @@ local ($file, $ids, $headersonly) = @_;
 local $retried = $_[3];
 &mark_read_maildir($file);
 local @files = &get_maildir_files($file);
+my $maps = &get_maildir_files_maps($file, \@files);
+
 local @rv;
 local $missing;
 foreach my $i (@$ids) {
@@ -2135,10 +2136,15 @@ foreach my $i (@$ids) {
 	local $mail = &read_mail_file($path, $headersonly);
 	if (!$mail && $path =~ /^(.*)\/(cur|tmp|new)\/([^:]*)(:2,([A-Za-z]*))?$/) {
 		# Flag may have changed - update path
-		local $suffix = "$2/$3";
-		local ($newfile) = grep
-		  { substr($_, length($file)+1, length($suffix)) eq $suffix }
-		  @files;
+		my $base = "$1/$2/$3";
+		if (!$maps->{'base'}) {
+			foreach my $f (@files) {
+				my $fb = $f;
+				$fb =~ s/:2,[A-Za-z]*$//;
+				$maps->{'base'}->{$fb} ||= $f;
+				}
+			}
+		local $newfile = $maps->{'base'}->{$base};
 		if ($newfile) {
 			$path = $newfile;
 			$mail = &read_mail_file($path, $headersonly);
@@ -2154,7 +2160,8 @@ foreach my $i (@$ids) {
 		$mail->{'id'} = $path;
 		$mail->{'id'} = substr($mail->{'id'}, length($file)+1);
 		# Get index in directory
-		$mail->{'idx'} = &indexof($path, @files);
+		my $idx = $maps->{'idx'}->{$path};
+		$mail->{'idx'} = defined($idx) ? $idx : -1;
 		}
 	else {
 		$missing = 1;
@@ -2168,6 +2175,27 @@ if ($missing && !$retried) {
 return @rv;
 }
 
+# get_maildir_files_maps(directory, &files)
+# Returns a hash ref containing a map from the paths of files in a Maildir to
+# their indexes, so that lookups don't need to scan the whole list. A map from
+# paths without flags to current paths is added by the caller if needed. The
+# maps are re-used until the cached list of files changes.
+sub get_maildir_files_maps
+{
+my ($dir, $files) = @_;
+my $cache = $main::list_maildir_cache{$dir};
+my $maps = $main::list_maildir_maps{$dir};
+if (!$maps || !$cache || $maps->{'cache'} ne $cache) {
+	my %idx;
+	for(my $i=0; $i<@$files; $i++) {
+		$idx{$files->[$i]} = $i;
+		}
+	$maps = { 'idx' => \%idx, 'cache' => $cache };
+	$main::list_maildir_maps{$dir} = $maps if ($cache);
+	}
+return $maps;
+}
+
 # Get ordered list of message files (with in-memory and on-disk caching, as
 # this can be slow)
 # get_maildir_files(directory)
@@ -2199,7 +2227,6 @@ else {
 			push(@files, $_[0]."/".$_);
 			}
 		close(CACHE);
-		$main::list_maildir_cache_time{$_[0]} = $cst[9];
 		}
 	else {
 		# Really read
@@ -2261,9 +2288,9 @@ else {
 							   undef, $cachefile);
 				}
 			}
-		$main::list_maildir_cache_time{$_[0]} = $st[9];
 		}
 	$main::list_maildir_cache{$_[0]} = \@files;
+	$main::list_maildir_cache_time{$_[0]} = $newest;
 	}
 return @files;
 }
@@ -2301,37 +2328,90 @@ sub mark_read_maildir
 local ($dir) = @_;
 local @files = &get_maildir_files($dir);
 local $i = 0;
+local $changed = 0;
+local $stale = 0;
 foreach my $nf (@files) {
 	if (substr($nf, length($dir)+1, 3) eq "new") {
 		local $cf = $nf;
-		$cf =~ s/\/new\//\/cur\//g;
-		if (&rename_as_mail_user($nf, $cf)) {
+		$cf =~ s/\/new\/([^\/]+)$/\/cur\/$1/;
+		if (!-e $nf) {
+			# Already moved or deleted by another process, so the
+			# cached file list is out of date. Don't try to rename,
+			# as rename_file would fall back to running mv
+			$stale = 1;
+			}
+		elsif (&rename_as_mail_user($nf, $cf)) {
 			$files[$i] = $cf;
 			$changed = 1;
 			}
 		}
 	$i++;
 	}
-if ($changed) {
-	# Update the cache
-	$main::list_maildir_cache{$dir} = \@files;
-	local $cachefile = &get_maildir_cachefile($dir);
-	if ($cachefile) {
-		&open_tempfile(CACHE, ">$cachefile", 1);
-		foreach my $f (@files) {
-			local $short = substr($f, length($dir)+1);
-			&print_tempfile(CACHE, $short,"\n");
-			}
-		&close_tempfile(CACHE);
-		local @st = stat($_[0]);
-		if ($< == 0) {
-			&set_ownership_permissions($st[4], $st[5],
-						   undef, $cachefile);
-			}
+if ($stale) {
+	# Force a re-read of the directory next time
+	&flush_maildir_cachefile($dir);
+	}
+elsif ($changed) {
+	&save_maildir_cachefile($dir, \@files);
+	}
+}
+
+# save_maildir_cachefile(dir, &files)
+# Updates the in-memory and on-disk caches of files in a Maildir after this
+# process has renamed some of them, including the new directory modification
+# time so that the caches remain valid
+sub save_maildir_cachefile
+{
+my ($dir, $files) = @_;
+my $newest;
+foreach my $d ("$dir/cur", "$dir/new") {
+	my @dst = stat($d);
+	$newest = $dst[9] if ($dst[9] > $newest);
+	}
+$main::list_maildir_cache{$dir} = $files;
+$main::list_maildir_cache_time{$dir} = $newest;
+my $cachefile = &get_maildir_cachefile($dir);
+if ($cachefile) {
+	&open_tempfile(CACHE, ">$cachefile", 1);
+	foreach my $f (@$files) {
+		my $short = substr($f, length($dir)+1);
+		&print_tempfile(CACHE, $short,"\n");
+		}
+	&close_tempfile(CACHE);
+	my @st = stat($dir);
+	if ($< == 0) {
+		&set_ownership_permissions($st[4], $st[5], undef, $cachefile);
 		}
 	}
 }
 
+# rename_maildir_cachefile(dir, old-path, new-path)
+# Updates the caches of files in a Maildir after one is renamed by this
+# process, such as when its flags change. Returns 1 if the caches were up to
+# date and could be updated, or 0 if they were flushed instead.
+sub rename_maildir_cachefile
+{
+my ($dir, $oldpath, $newpath) = @_;
+my $cache = $main::list_maildir_cache{$dir};
+my $maps = $main::list_maildir_maps{$dir};
+my $i;
+if ($cache && $maps && $maps->{'cache'} eq $cache) {
+	$i = $maps->{'idx'}->{$oldpath};
+	}
+elsif ($cache) {
+	$i = &indexof($oldpath, @$cache);
+	$i = undef if ($i < 0);
+	}
+if (!defined($i)) {
+	&flush_maildir_cachefile($dir);
+	return 0;
+	}
+my @files = @$cache;
+$files[$i] = $newpath;
+&save_maildir_cachefile($dir, \@files);
+return 1;
+}
+
 # delete_maildir(&mail, ...)
 # Delete messages from a maildir directory
 sub delete_maildir

mailboxes/folders-lib.pl

@@ -572,20 +572,22 @@ foreach my $mail (@mail) {
 return @mail;
 }
 
-# mailbox_get_mail(&folder, id, headersonly)
-# Convenience function to get a single mail by ID
+# mailbox_get_mail(&folder, id, headersonly, [skip-sort-index])
+# Convenience function to get a single mail by ID. Unless skip-sort-index is
+# set, the message's position in the sorted folder is also computed, which
+# can be slow for large folders.
 sub mailbox_get_mail
 {
-local ($folder, $id, $headersonly) = @_;
+local ($folder, $id, $headersonly, $nosortidx) = @_;
 local ($mail) = &mailbox_select_mails($folder, [ $id ], $headersonly);
-if ($mail) {
+if ($mail && !$nosortidx) {
 	# Find the sort index for this message
 	local ($field, $dir) = &get_sort_field($folder);
 	if (!$field || !$folder->{'sortable'}) {
 		# No sorting, so sort index is the opposite of real
 		$mail->{'sortidx'} = &mailbox_folder_size($folder, 1) -
 				     $mail->{'idx'} - 1;
-		print DEBUG "idx=$mail->{'idx'} sortidx=$mail->{'sortidx'} size=",&mailbox_folder_size($folder, 1),"\n";
+		print DEBUG "idx=$mail->{'idx'} sortidx=$mail->{'sortidx'}\n";
 		}
 	else {
 		# Need to extract from sort index
@@ -643,10 +645,20 @@ elsif ($folder->{'type'} == 4) {
 	return () if (!$count);
         $folder->{'lastchange'} = $rv[3] if ($rv[3]);
 
-	@rv = &imap_command($h, "FETCH 1:$count UID");
-	foreach my $uid (@{$rv[1]}) {
-		if ($uid =~ /UID\s+(\d+)/) {
-			push(@idlist, $1);
+	# UID SEARCH returns all UIDs on one line, which is much faster to
+	# read than one FETCH response per message
+	@rv = &imap_command($h, "UID SEARCH ALL");
+	if ($rv[0]) {
+		local ($srch) = grep { $_ =~ /^\*\s+SEARCH/i } @{$rv[1]};
+		$srch =~ s/^\*\s+SEARCH\s*//i;
+		@idlist = sort { $a <=> $b } grep { /^\d+$/ } split(/\s+/, $srch);
+		}
+	else {
+		@rv = &imap_command($h, "FETCH 1:$count UID");
+		foreach my $uid (@{$rv[1]}) {
+			if ($uid =~ /UID\s+(\d+)/) {
+				push(@idlist, $1);
+				}
 			}
 		}
 	}
@@ -778,15 +790,36 @@ local ($folder, $field, $dir) = @_;
 # Build or update the sort index. This is a file mapping unique IDs and fields
 # to sortable values.
 local %index;
-&build_new_sort_index($folder, $field, \%index);
+&build_new_sort_index($folder, $field, \%index) || return ( );
 
-# Get message indexes, sorted by the field
+# Re-use the sorted list if already built by this process, and the index
+# hasn't changed since
+local $ifile = &folder_new_sort_index_file($folder);
+my $memo = $main::sorted_ids_cache{$ifile};
+if ($memo && $memo->{'field'} eq $field && $memo->{'dir'} eq $dir) {
+	return @{$memo->{'ids'}};
+	}
+
+# Get message indexes and the values of the field. If the list of IDs in the
+# index is known, fetch each value directly as this is much faster than
+# iterating over the whole DBM.
 my @sorter;
-while(my ($k, $v) = each %index) {
-	if ($k =~ /^(.*)_\Q$field\E$/) {
-		push(@sorter, [ $1, lc($v) ]);
+my $ids = &read_sort_index_ids($ifile, \%index);
+if ($ids) {
+	foreach my $id (@$ids) {
+		my $v = $index{$id."_".$field};
+		push(@sorter, [ $id, lc($v) ]) if (defined($v));
 		}
 	}
+else {
+	while(my ($k, $v) = each %index) {
+		if ($k =~ /^(.*)_\Q$field\E$/) {
+			push(@sorter, [ $1, lc($v) ]);
+			}
+		}
+	}
+
+# Sort by the field
 if ($field eq "size" || $field eq "date" || $field eq "x-spam-status") {
 	# Numeric sort
 	@sorter = sort { my $s = $a->[1] <=> $b->[1]; $dir ? $s : -$s } @sorter;
@@ -795,7 +828,47 @@ else {
 	# Alpha sort
 	@sorter = sort { my $s = $a->[1] cmp $b->[1]; $dir ? $s : -$s } @sorter;
 	}
-return map { $_->[0] } @sorter;
+my @rv = map { $_->[0] } @sorter;
+$main::sorted_ids_cache{$ifile} = { 'field' => $field, 'dir' => $dir,
+				    'ids' => \@rv };
+return @rv;
+}
+
+# read_sort_index_ids(index-file, &index)
+# Returns an array ref of message IDs in a sort index, from the list saved
+# alongside it, or undef if the list is missing or out of date
+sub read_sort_index_ids
+{
+my ($ifile, $index) = @_;
+my $fh;
+open($fh, "<", "$ifile.ids") || return undef;
+my $hdr = <$fh>;
+chomp($hdr);
+if ($hdr ne $index->{'lastchange'}." ".$index->{'mailcount'}) {
+	close($fh);
+	return undef;
+	}
+my @ids = <$fh>;
+close($fh);
+chomp(@ids);
+return \@ids;
+}
+
+# write_sort_index_ids(index-file, &index, &ids)
+# Saves the list of message IDs in a sort index, along with its last update
+# time and message count so that it can be checked for validity
+sub write_sort_index_ids
+{
+my ($ifile, $index, $ids) = @_;
+my $fh = "IDSFILE";
+&open_tempfile($fh, ">$ifile.ids", 1) || return 0;
+&print_tempfile($fh, $index->{'lastchange'}." ".$index->{'mailcount'}."\n");
+foreach my $id (@$ids) {
+	&print_tempfile($fh, $id."\n");
+	}
+my $ok = &close_tempfile($fh);
+chmod(0600, "$ifile.ids");
+return $ok;
 }
 
 # delete_old_sort_index(&folder)
@@ -880,24 +953,40 @@ if ($index->{'lastchange'} != $folder->{'lastchange'} ||
 		}
 	print DEBUG "added ",scalar(@mails)," messages to index\n";
 
-	# Remove IDs that no longer exist
+	# Remove IDs that no longer exist. If the list of IDs previously in
+	# the index is known, just delete their entries, rather than
+	# iterating over the whole DBM to find them.
 	local %ids = map { $_, 1 } (@ids, @wantids);
 	local $dc = 0;
-	local @todelete;
-	while(my ($k, $v) = each %$index) {
-		if ($k =~ /^(.*)_([^_]+)$/ && !$ids{$1}) {
-			push(@todelete, $k);
-			$dc++ if ($2 eq "size");
+	my $oldids = &read_sort_index_ids($ifile, $index);
+	if ($oldids) {
+		foreach my $id (@$oldids) {
+			next if ($ids{$id});
+			foreach my $f (@index_fields) {
+				delete($index->{$id."_".$f});
+				}
+			$dc++;
 			}
 		}
-	foreach my $k (@todelete) {
-		delete($index->{$k});
+	else {
+		local @todelete;
+		while(my ($k, $v) = each %$index) {
+			if ($k =~ /^(.*)_([^_]+)$/ && !$ids{$1}) {
+				push(@todelete, $k);
+				$dc++ if ($2 eq "size");
+				}
+			}
+		foreach my $k (@todelete) {
+			delete($index->{$k});
+			}
 		}
 	print DEBUG "deleted $dc messages from index\n";
 
-	# Record index update time
+	# Record index update time, and the IDs now in it
 	$index->{'lastchange'} = $folder->{'lastchange'} || time();
 	$index->{'mailcount'} = scalar(@ids);
+	&write_sort_index_ids($ifile, $index, [ &unique(@ids, @wantids) ]);
+	delete($main::sorted_ids_cache{$ifile});
 	print DEBUG "new indexchange=$index->{'lastchange'}\n";
 	}
 return 1;
@@ -914,6 +1003,7 @@ foreach my $field (@index_fields) {
 	delete($index{$id."_".$field});
 	}
 dbmclose(%index);
+delete($main::sorted_ids_cache{&folder_new_sort_index_file($folder)});
 if ($folder->{'type'} == 5 || $folder->{'type'} == 6) {
 	# Remove from underlying folder's index too
 	local ($sfn, $sid) = split(/\t+/, $id, 2);
@@ -933,6 +1023,50 @@ local %index;
 &build_new_sort_index($folder, undef, \%index);
 $index{'lastchange'} = 0;
 dbmclose(%index);
+delete($main::sorted_ids_cache{&folder_new_sort_index_file($folder)});
+}
+
+# rename_new_sort_index_message(&folder, old-id, new-id)
+# Updates a sort index after a message's ID changes, such as when the flags in
+# a Maildir filename change, so that it doesn't need to be re-read
+sub rename_new_sort_index_message
+{
+my ($folder, $oldid, $newid) = @_;
+return 0 if (!$folder->{'sortable'});
+my $ifile = &folder_new_sort_index_file($folder);
+my %index;
+&open_dbm_db(\%index, $ifile, 0600);
+if (defined($index{$oldid."_size"})) {
+	foreach my $f (@index_fields) {
+		$index{$newid."_".$f} = $index{$oldid."_".$f};
+		delete($index{$oldid."_".$f});
+		}
+	my $ids = &read_sort_index_ids($ifile, \%index);
+	if ($ids) {
+		my $i = &indexof($oldid, @$ids);
+		if ($i >= 0) {
+			$ids->[$i] = $newid;
+			&write_sort_index_ids($ifile, \%index, $ids);
+			}
+		else {
+			unlink("$ifile.ids");
+			}
+		}
+	}
+dbmclose(%index);
+
+# Update the ID in any already-sorted list too, as its position is unchanged
+my $memo = $main::sorted_ids_cache{$ifile};
+if ($memo) {
+	my $i = &indexof($oldid, @{$memo->{'ids'}});
+	if ($i >= 0) {
+		$memo->{'ids'}->[$i] = $newid;
+		}
+	else {
+		delete($main::sorted_ids_cache{$ifile});
+		}
+	}
+return 1;
 }
 
 # delete_new_sort_index(&folder)
@@ -945,6 +1079,9 @@ local $ifile = &folder_new_sort_index_file($folder);
 my %index;
 &open_dbm_db(\%index, $ifile, 0600);
 %index = ( );
+dbmclose(%index);
+unlink("$ifile.ids");
+delete($main::sorted_ids_cache{$ifile});
 }
 
 # folder_sort_index_file(&folder)
@@ -1853,7 +1990,7 @@ elsif ($f->{'type'} == 6 && !$est) {
 	# exist in the parent folders
 	$rv = 0;
 	foreach my $msg (@{$f->{'members'}}) {
-		if (&mailbox_get_mail($msg->[0], $msg->[1])) {
+		if (&mailbox_get_mail($msg->[0], $msg->[1], 1, 1)) {
 			$rv++;
 			}
 		}
@@ -1904,12 +2041,19 @@ elsif ($folder->{'type'} == 5) {
 	}
 else {
 	# For all other folders, just check individual messages
-	# XXX faster for maildir?
 	local @data = ( 0, 0, 0 );
 	local @mails;
 	eval {
 		$main::error_must_die = 1;
-		@mails = &mailbox_list_mails(undef, undef, $folder, 1);
+		if ($folder->{'type'} == 1 && $folder->{'flags'}) {
+			# For a Maildir where read flags come from filenames,
+			# only read the headers of messages with no flags set,
+			# as only they need the read DBM checked
+			@mails = &list_maildir_flags($folder);
+			}
+		else {
+			@mails = &mailbox_list_mails(undef, undef, $folder, 1);
+			}
 		};
 	return ( ) if ($@);
 	foreach my $m (@mails) {
@@ -1926,6 +2070,36 @@ else {
 	}
 }
 
+# list_maildir_flags(&folder)
+# Returns messages in a Maildir folder with IDs and flags taken from their
+# filenames. Headers are only read for messages with no flags.
+sub list_maildir_flags
+{
+my ($folder) = @_;
+my $md = $folder->{'file'};
+my @rv;
+&switch_to_folder_user($folder);
+&mark_read_maildir($md);
+foreach my $f (&get_maildir_files($md)) {
+	my $mail = { 'header' => { } };
+	if ($f =~ /:2,([A-Za-z]*)$/) {
+		my @flags = split(//, $1);
+		$mail->{'read'} = &indexoflc("S", @flags) >= 0 ? 1 : 0;
+		$mail->{'special'} = &indexoflc("F", @flags) >= 0 ? 1 : 0;
+		$mail->{'replied'} = &indexoflc("R", @flags) >= 0 ? 1 : 0;
+		$mail->{'flags'} = 1;
+		}
+	if (!$mail->{'read'} && !$mail->{'special'} && !$mail->{'replied'}) {
+		$mail = &read_mail_file($f, 1) || next;
+		}
+	$mail->{'file'} = $f;
+	$mail->{'id'} = substr($f, length($md)+1);
+	push(@rv, $mail);
+	}
+&switch_from_folder_user($folder);
+return @rv;
+}
+
 # mailbox_set_read_flags(&folder, &mail, read, special, replied)
 # Updates the status flags on some message
 sub mailbox_set_read_flag
@@ -1970,11 +2144,23 @@ elsif ($folder->{'type'} == 1) {
 	my $newfile = $base.":2,".
 			 join("", grep { $flags{$_} } sort(keys %flags));
 	if ($newfile ne $file) {
-		# Need to rename file
-		rename("$path$file", "$path$newfile");
+		# Need to rename file. Make sure the cached list of files is
+		# current first, so that it can be updated rather than
+		# flushed, which would force a re-read of the whole directory
+		&get_maildir_files($folder->{'file'});
+		my $oldid = $mail->{'id'};
+		if (rename("$path$file", "$path$newfile")) {
+			&rename_maildir_cachefile($folder->{'file'},
+				"$path$file", "$path$newfile");
+			}
+		else {
+			&flush_maildir_cachefile($folder->{'file'});
+			}
+		$mail->{'file'} = "$path$newfile" if ($mail->{'file'});
 		$newfile =~ s/^(.*)\/((cur|tmp|new)\/.*)$/$2/;
 		$mail->{'id'} = $newfile;
-		&flush_maildir_cachefile($folder->{'file'});
+		&rename_new_sort_index_message($folder, $oldid, $newfile)
+			if ($oldid ne $newfile);
 		}
 	}
 else {

mailboxes/mailboxes-lib.pl

@@ -1143,6 +1143,7 @@ foreach my $folder (&list_user_folders($user)) {
 	else {
 		&unlink_file(glob("\Q$ifile\E.{dir,pag,db}"));
 		}
+	&unlink_file("$ifile.ids") if (-e "$ifile.ids");
 	}
 # Remove read file
 local $read = &user_read_dbm_file($user);

miniserv-lib.pl

@@ -1565,8 +1565,23 @@ if (&get_type($full) eq "internal/cgi" && $validated != 4) {
 				$cgiheader{lc($1)} = $2;
 				push(@cgiheader, [ $1, $2 ]);
 				}
+			# A CGI can reject a request before producing normal
+			# output. Use its explicit status instead of the default
+			# success code.
+			my $cgi_status = $cgiheader{'status'} =~
+				/\A([1-5][0-9][0-9])\s+([^\r\n]+)\z/;
+			if ($cgi_status) {
+				($ok_code, $ok_message) = ($1, $2);
+				}
+			else {
+				# Invalid control headers must not affect
+				# redirect logging.
+				delete($cgiheader{'status'});
+				}
 			if ($cgiheader{"location"}) {
-				&write_data("HTTP/1.0 302 Moved Temporarily\r\n");
+				&write_data("HTTP/1.0 ".($cgi_status
+					? "$ok_code $ok_message"
+					: "302 Moved Temporarily")."\r\n");
 				&write_data("Date: $datestr\r\n");
 				&write_data("Server: @{[&server_info()]}\r\n");
 				&write_keep_alive(0);
@@ -1588,6 +1603,7 @@ if (&get_type($full) eq "internal/cgi" && $validated != 4) {
 				&write_keep_alive(0);
 				}
 			foreach $h (@cgiheader) {
+				next if (lc($h->[0]) eq 'status');
 				&write_data("$h->[0]: $h->[1]\r\n");
 				}
 			&write_data("\r\n");
@@ -1671,7 +1687,9 @@ else {
 # log the request
 &log_request($loghost, $authuser, $reqline,
 	     $logged_code ? $logged_code :
-	     $cgiheader{"location"} ? "302" : $ok_code, &byte_count());
+	     $cgiheader{"location"} && !$cgiheader{"status"}
+		? "302"
+		: $ok_code, &byte_count());
 return $rv;
 }
 
@@ -2499,9 +2517,22 @@ foreach $d (@_) {
 				}
 			}
 		if ($doneheaders) {
+			# Match forked CGI status handling for in-process
+			# scripts.
+			my $cgi_status = $cgiheader{'status'} =~
+				/\A([1-5][0-9][0-9])\s+([^\r\n]+)\z/;
+			if ($cgi_status) {
+				($ok_code, $ok_message) = ($1, $2);
+				}
+			else {
+				# Invalid control headers must not affect
+				# redirect logging.
+				delete($cgiheader{'status'});
+				}
 			if ($cgiheader{"location"}) {
-				&write_data(
-					"HTTP/1.0 302 Moved Temporarily\r\n");
+				&write_data("HTTP/1.0 ".($cgi_status
+					? "$ok_code $ok_message"
+					: "302 Moved Temporarily")."\r\n");
 				&write_data("Date: $datestr\r\n");
 				&write_data("Server: @{[&server_info()]}\r\n");
 				&write_keep_alive(0);
@@ -2516,6 +2547,7 @@ foreach $d (@_) {
 				&write_keep_alive(0);
 				}
 			foreach $h (@cgiheader) {
+				next if (lc($h->[0]) eq 'status');
 				&write_data("$h->[0]: $h->[1]\r\n");
 				}
 			&write_data("\r\n");
@@ -2553,7 +2585,9 @@ if ($doing_cgi_eval && $$ == $main_process_id) {
 	close(SOCK);
 	close($PASSINw); close($PASSOUTw);
 	&log_request($loghost, $authuser, $reqline,
-		     $cgiheader{"location"} ? "302" : $ok_code, &byte_count());
+		     $cgiheader{"location"} && !$cgiheader{"status"}
+		     	? "302"
+			: $ok_code, &byte_count());
 	}
 }
 

mysql/backup_form.cgi

@@ -33,7 +33,8 @@ if ($in{'all'}) {
 	print "$text{'backup_desc3'}\n";
 	}
 else {
-	print &text('backup_desc', "<tt>".&html_escape($in{'db'})."</tt>"),"\n";
+	print &text('backup_desc',
+		    "<tt>".&html_escape($in{'db'})."</tt>"),"\n";
 	}
 if ($cron) {
 	print "$text{'backup_desc2'}\n";

mysql/edit_dbase.cgi

@@ -29,11 +29,12 @@ if ($in{'search'}) {
 	@indexes = grep { /\Q$in{'search'}\E/i } @indexes;
 	@views = grep { /\Q$in{'search'}\E/i } @views;
 	print "<table width=100%><tr>\n";
-	print "<td> <b>",&text('dbase_showing',
-		"<tt>".&html_escape($in{'search'})."</tt>"),"</b></td>\n";
-	print "<td align=right><a href='edit_dbase.cgi?db=",
-		&urlize($in{'db'}),"'>",
-		"$text{'view_searchreset'}</a></td>\n";
+	print "<td> <b>",
+	      &text('dbase_showing',
+	  	    "<tt>".&html_escape($in{'search'})."</tt>"),
+	      "</b></td>\n";
+	print "<td align=right><a href='edit_dbase.cgi?db=".&urlize($in{'db'})."'>".
+	      "$text{'view_searchreset'}</a></td>\n";
 	print "</tr></table>\n";
 	}
 
@@ -83,12 +84,12 @@ elsif (@titles || @indexes) {
 		   ( map { "images/index.gif" } @indexes ),
 		   ( map { "images/view.gif" } @views ),
 		 );
-	@links = ( ( map { "edit_table.cgi?db=".&urlize($in{'db'})."&table=".&urlize($_) }
-		     	 @titles ),
-		   ( map { "edit_index.cgi?db=".&urlize($in{'db'})."&index=".&urlize($_) }
-                         @indexes ),
-		   ( map { "edit_view.cgi?db=".&urlize($in{'db'})."&view=".&urlize($_) }
-                         @views ),
+	@links = ( ( map { "edit_table.cgi?db=".&urlize($in{'db'}).
+			   "&table=".&urlize($_) } @titles ),
+		   ( map { "edit_index.cgi?db=".&urlize($in{'db'}).
+			   "&index=".&urlize($_) } @indexes ),
+		   ( map { "edit_view.cgi?db=".&urlize($in{'db'}).
+			   "&view=".&urlize($_) } @views ),
 		 );
 	@descs = ( ( map { "" } @titles ),
 		   ( map { " ($text{'dbase_index'})" } @indexes),

mysql/view-lib.pl

@@ -111,7 +111,7 @@ if (&foreign_check("virtual-server") &&
 			my ($got) = grep { $_->{'name'} eq $dbname } @dbs;
 			if ($got) {
 				return "../virtual-server/list_databases.cgi?".
-				       "dom=$d->{'id'}";
+				       "dom=".&urlize($d->{'id'});
 				}
 			}
 		}

mysql/view_table.cgi

@@ -209,23 +209,27 @@ if ($in{'field'}) {
 	# Show details of simple search
 	my $msg = $in{'match'} == 2 || $in{'match'} == 3 ?
 			'view_searchheadnot' : 'view_searchhead';
-	print "<tr> <td><b>",&text($msg, "<tt>".&html_escape($in{'for'})."</tt>",
-			   "<tt>".&html_escape($in{'field'})."</tt>"),"</b></td>\n";
-	print "<td align=right><a href='view_table.cgi?db=",&urlize($in{'db'}),
-	      "&table=",&urlize($in{'table'}),"$sortargs'>$text{'view_searchreset'}</a></td> </tr>\n";
+	print "<tr> <td><b>",&text($msg,
+			"<tt>".&html_escape($in{'for'})."</tt>",
+			"<tt>".&html_escape($in{'field'})."</tt>"),"</b></td>\n";
+	print "<td align=right><a href='view_table.cgi?db=".&urlize($in{'db'})."&".
+	      "table=".&urlize($in{'table'})."$sortargs'>".
+	      "$text{'view_searchreset'}</a></td> </tr>\n";
 	}
 elsif ($in{'advanced'}) {
 	# Show details of advanced search
 	print "<tr> <td><b>",&text('view_searchhead2', $advcount),"</b></td>\n";
-	print "<td align=right><a href='view_table.cgi?db=",&urlize($in{'db'}),
-	      "&table=",&urlize($in{'table'}),"$sortargs'>$text{'view_searchreset'}</a></td> </tr>\n";
+	print "<td align=right><a href='view_table.cgi?db=".&urlize($in{'db'})."&".
+	      "table=".&urlize($in{'table'})."$sortargs'>".
+	      "$text{'view_searchreset'}</a></td> </tr>\n";
 	}
 if ($in{'sortfield'}) {
 	# Show current sort order
 	print "<tr> <td><b>",&text($in{'sortdir'} ? 'view_sorthead2' : 'view_sorthead1',
 			      "<tt>".&html_escape($in{'sortfield'})."</tt>"),"</b></td>\n";
-	print "<td align=right><a href='view_table.cgi?db=",&urlize($in{'db'}),
-	      "&table=",&urlize($in{'table'}),"$searchargs'>$text{'view_sortreset'}</a></td> </tr>\n";
+	print "<td align=right><a href='view_table.cgi?db=".&urlize($in{'db'})."&".
+	      "table=".&urlize($in{'table'})."$searchargs'>".
+	      "$text{'view_sortreset'}</a></td> </tr>\n";
 	}
 
 print "</table>\n";
@@ -398,7 +402,7 @@ if ($total || $in{'new'}) {
 						  "$module_name/download.cgi?".
 						  "db=".&urlize($in{'db'}).
 						  "&table=".&urlize($in{'table'}).
-						  "&start=$in{'start'}".
+						  "&start=".&urlize($in{'start'}).
 						  $searchargs.$sortargs.
 						  "&row=$i&col=$j",
 						$text{'view_download'}));
@@ -492,15 +496,18 @@ if (!$in{'field'} && $total > $displayconfig{'perpage'}) {
 	}
 
 if ($access{'edonly'}) {
-	&ui_print_footer("edit_dbase.cgi?db=".&urlize($in{'db'}),$text{'dbase_return'},
-		 &get_databases_return_link($in{'db'}), $text{'index_return'});
+	&ui_print_footer(
+		"edit_dbase.cgi?db=".&urlize($in{'db'}), $text{'dbase_return'},
+		&get_databases_return_link($in{'db'}), $text{'index_return'});
 	}
 else {
-	&ui_print_footer("edit_table.cgi?db=".&urlize($in{'db'})."&table=".
-			 &urlize($in{'table'}),
-			$text{'table_return'},
-			"edit_dbase.cgi?db=".&urlize($in{'db'}), $text{'dbase_return'},
-			&get_databases_return_link($in{'db'}), $text{'index_return'});
+	&ui_print_footer("edit_table.cgi?db=".&urlize($in{'db'}).
+			 "&table=".&urlize($in{'table'}),
+			 	$text{'table_return'},
+			"edit_dbase.cgi?db=".&urlize($in{'db'}),
+				$text{'dbase_return'},
+			&get_databases_return_link($in{'db'}),
+				$text{'index_return'});
 	}
 
 # column_sort_link(name)
@@ -512,9 +519,9 @@ local $dir = $in{'sortfield'} eq $field ? !$in{'sortdir'} : 0;
 local $img = $in{'sortfield'} eq $field && $dir ? "sortascgrey.gif" :
 	     $in{'sortfield'} eq $field && !$dir ? "sortdescgrey.gif" :
 	     $dir ? "sortasc.gif" : "sortdesc.gif";
-return "<a href='view_table.cgi?db=".&urlize($in{'db'})."&table=".
-       &urlize($in{'table'})."&start=$in{'start'}&sortfield=".
-       &urlize($field)."&sortdir=$dir$searchargs'>".
+return "<a href='view_table.cgi?db=".&urlize($in{'db'}).
+       "&table=".&urlize($in{'table'})."&start=".&urlize($in{'start'}).
+       "&sortfield=".&urlize($field)."&sortdir=".&urlize($dir)."$searchargs'>".
        "<b>".&html_escape($field)."</b><img valign=middle src=../images/$img border=0>";
 }
 

postgresql/lang/en

@@ -159,6 +159,7 @@ exec_tabimport=Import text file
 exec_return=execute SQL form
 exec_ecompress=The SQL file is compressed, but the program $1 needed to un-compress it is not installed.
 exec_ecompress2=Un-compression failed : $1
+exec_erestrict=The psql client is too old to safely execute SQL files. Update the PostgreSQL client and try again.
 
 stop_err=Failed to stop database server
 stop_epidfile=Failed to open PID file $1

postgresql/postgresql-lib.pl

@@ -373,10 +373,7 @@ else {
 		}
         elsif ($sql =~ /^\s*\\/ ) {
 		$break_f = 1 ;
-		if ($sql !~ /^\s*\\copy\s+/ &&
-                    $sql !~ /^\s*\\i\s+/) {
-			&error ( &text ( 'r_command', ) ) ;
-			}
+		&error ( &text ( 'r_command', ) ) ;
 		}
 
 	if (@params) {
@@ -832,6 +829,22 @@ return 1 if (!$user);
 return &set_ownership_permissions($user, undef, 0600, $file);
 }
 
+# supports_psql_restrict()
+# Returns 1 if psql supports restricted mode.
+sub supports_psql_restrict
+{
+local $out = &backquote_command(
+	&quote_path($config{'psql'})." -V 2>&1 <$null_file");
+local ($version) = $out =~ /psql.*?\s(\d+(?:\.\d+)*)/i;
+return 0 if (!defined($version));
+local ($major) = split(/\./, $version);
+local %minimum = ( 13 => "13.22", 14 => "14.19", 15 => "15.14",
+		  16 => "16.10", 17 => "17.6" );
+return &compare_version_numbers($version, "18") >= 0 ||
+       $minimum{$major} &&
+       &compare_version_numbers($version, $minimum{$major}) >= 0;
+}
+
 # execute_sql_file(database, file, [user, pass], [unix-user])
 # Executes some file of SQL statements, and returns the exit status and output
 sub execute_sql_file
@@ -844,7 +857,59 @@ if (!defined($user)) {
 	$user = $postgres_login;
 	$pass = $postgres_pass;
 	}
-local $cmd = &quote_path($config{'psql'})." -f ".&quote_path($file).
+&supports_psql_restrict() || return (1, $text{'exec_erestrict'});
+
+# Copy a plain PostgreSQL dump without its outer restriction wrapper.
+local (*SQLFILE, *SAFEFILE);
+local ($line, $dump_key);
+open(SQLFILE, "<$file") || return (1, $text{'exec_efile'});
+while(defined($line = <SQLFILE>)) {
+	if ($line =~ /^\s*\\restrict\s+([A-Za-z0-9]+)\s*$/) {
+		$dump_key = $1;
+		last;
+		}
+	elsif ($line !~ /^\s*(?:--.*)?$/) {
+		last;
+		}
+	}
+local $safe_file;
+if ($dump_key) {
+	$safe_file = &transname();
+	&open_tempfile(SAFEFILE, ">$safe_file");
+	local ($last_line, $last_pos);
+	while(defined($line = <SQLFILE>)) {
+		if ($line =~ /\S/) {
+			$last_line = $line;
+			$last_pos = tell(SAFEFILE);
+			}
+		&print_tempfile(SAFEFILE, $line);
+		}
+	&close_tempfile(SAFEFILE);
+	if (!defined($last_line) ||
+	    $last_line !~ /^\s*\\unrestrict\s+\Q$dump_key\E\s*$/ ||
+	    !truncate($safe_file, $last_pos)) {
+		unlink($safe_file);
+		undef($safe_file);
+		}
+	}
+close(SQLFILE);
+if ($safe_file) {
+	local $fileuser = &postgresql_command_unix_user();
+	$fileuser ||= $unixuser if ($unixuser && $unixuser ne 'root' && $< == 0);
+	if ($fileuser &&
+	    !&set_ownership_permissions($fileuser, undef, 0600, $safe_file)) {
+		unlink($safe_file);
+		return (1, $text{'exec_efile'});
+		}
+	$file = $safe_file;
+	}
+
+# Restrict psql commands while executing the user-supplied file.
+local $restrict_key = &generate_miniserv_websocket_token();
+local $cmd = &quote_path($config{'psql'}).
+	     " -X -c ".
+	     quotemeta("\\restrict $restrict_key").
+	     " -f ".&quote_path($file).
 	     &host_port_flags().
 	     (&supports_pgpass() ? " -U $user" : " -u").
 	     " ".$db;
@@ -855,8 +920,12 @@ elsif ($unixuser && $unixuser ne 'root' && $< == 0) {
 	$cmd = &command_as_user($unixuser, 0, $cmd);
 	}
 $cmd = &command_with_login($cmd, $user, $pass);
+&clean_language();
 local $out = &backquote_logged("$cmd 2>&1");
-return ($out =~ /ERROR/i ? 1 : 0, $out);
+local $ex = $? ? 1 : 0;
+&reset_environment();
+unlink($safe_file) if ($safe_file);
+return ($ex || $out =~ /ERROR|backslash commands are restricted|\\unrestrict:\s+wrong key/i ? 1 : 0, $out);
 }
 
 # split_table(&titles, &checkboxes, &links, &col1, &col2, ...)

sendmail/edit_ffile.cgi

@@ -23,12 +23,13 @@ while(<FILE>) {
 	}
 close(FILE);
 
-print "<b>",&text('ffile_desc', "<tt>$in{'file'}</tt>"),"</b><p>\n";
+print "<b>",&text('ffile_desc',
+		  "<tt>".&html_escape($in{'file'})."</tt>"),"</b><p>\n";
 
-print "<form action=save_ffile.cgi method=post enctype=multipart/form-data>\n";
-print "<input type=hidden name=file value=\"$in{'file'}\">\n";
-print "<input type=hidden name=num value=\"$in{'num'}\">\n";
-print "<input type=hidden name=name value=\"$in{'name'}\">\n";
+print &ui_form_start("save_ffile.cgi", "form-data");
+print &ui_hidden("file", $in{'file'});
+print &ui_hidden("num", $in{'num'});
+print &ui_hidden("name", $in{'name'});
 
 $i = 0;
 foreach $f (@filter, [ 1, '', '', '' ],
@@ -36,33 +37,25 @@ foreach $f (@filter, [ 1, '', '', '' ],
 		     [ 1, '', '', '' ],
 		     [ 1, '', '', '' ],
 		     [ 1, '', '', '' ]) {
-	$field = "<select name=field_$i>\n";
-	foreach $ft ('', 'from', 'to', 'subject', 'cc', 'body') {
-		$field .= sprintf "<option value='%s' %s>%s</option>\n",
-			$ft, $f->[2] eq $ft ? "selected" : "",
-			$ft ? $text{"ffile_$ft"} : "&nbsp";
-		}
-	$field .= "</select>\n";
+	$field = &ui_select("field_$i", $f->[2],
+		[ map { [ $_, $text{'ffile_'.$_} ] }
+		      ('', 'from', 'to', 'subject', 'cc', 'body') ])."\n";
 
-	$what = "<select name=what_$i>\n";
-	$what .= sprintf "<option value=0 %s>%s</option>\n",
-		$f->[0] == 0 ? "selected" : "", $text{"ffile_what0"};
-	$what .= sprintf "<option value=1 %s>%s</option>\n",
-		$f->[0] == 1 ? "selected" : "", $text{"ffile_what1"};
-	$what .= "</select>\n";
+	$what = &ui_select("what_$i", $f->[0],
+			   [ [ 0, $text{'ffile_what0'} ],
+			     [ 1, $text{'ffile_what1'} ] ]);
 
-	$match = "<input name=match_$i size=20 value='$f->[3]'>\n";
+	$match = &ui_textbox("match_$i", $f->[3], 20)."\n";
 
-	$action = "<input name=action_$i size=30 value='$f->[1]'>\n";
+	$action = &ui_textbox("action_$i", $f->[1], 30)."\n";
 
 	print &text('ffile_line', $field, $what, $match, $action),"<br>\n";
 	$i++;
 	}
-print &text('ffile_other',
-	    "<input name=other size=30 value='$other'>"),"<br>\n";
+print &text('ffile_other', &ui_textbox("other", $other, 30)),"<br>\n";
 
-print "<input type=submit value=\"$text{'save'}\">\n";
-print "</form>\n";
+print &ui_form_end([ [ undef, $text{'save'} ] ]);
 
-&ui_print_footer("edit_alias.cgi?name=$in{'name'}&num=$in{'num'}", $text{'aform_return'});
+&ui_print_footer("edit_alias.cgi?name=".&urlize($in{'name'}).
+		 "&num=".&urlize($in{'num'}), $text{'aform_return'});
 

t/fixtures/miniserv-cgi-status.py

@@ -0,0 +1,126 @@
+"""Test CGI responses and request logs on a disposable Linux guest.
+
+Usage: python3 t/fixtures/miniserv-cgi-status.py /tmp/task-directory
+The source tree containing this fixture supplies miniserv.pl and miniserv-lib.pl.
+Each CGI mode gets a separate loopback server and temporary configuration.
+No Webmin account or installed configuration is used. Never run locally.
+"""
+from pathlib import Path
+import http.client
+import os
+import re
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+
+if sys.platform != 'linux':
+    raise RuntimeError('Run only inside a disposable Linux test guest')
+task = Path(sys.argv[1]).resolve()
+if not task.is_dir() or not any(p in task.parents for p in (Path('/tmp'), Path('/var/tmp'))):
+    raise RuntimeError('Supply an existing task directory under /tmp or /var/tmp')
+source = Path(__file__).resolve().parents[2]
+os.umask(0o077)
+
+
+def request(port, name):
+    """request(port, name): return status, headers and body without redirects."""
+    conn = http.client.HTTPConnection('127.0.0.1', port, timeout=10)
+    try:
+        conn.request('GET', '/' + name)
+        response = conn.getresponse()
+        return response.status, response.getheaders(), response.read().decode()
+    finally:
+        conn.close()
+
+
+def check(condition, message):
+    """check(condition, message): report a result or stop on failure."""
+    if not condition:
+        raise AssertionError(message)
+    print('PASS: ' + message, flush=True)
+
+
+# Cases cover defaults, explicit errors, redirects, malformed headers and exit.
+cases = [
+    ('normal', '', 200, False),
+    ('denied', 'Status: 403 Forbidden\r\n', 403, False),
+    ('created', 'Status: 201 Created\r\n', 201, False),
+    ('redirect', 'Location: /next\r\n', 302, False),
+    ('redirect307', 'Status: 307 Temporary Redirect\r\nLocation: /next\r\n', 307, False),
+    ('invalid', 'Status: invalid\r\nLocation: /next\r\n', 302, False),
+    ('badcode', 'Status: 999 Invalid\r\n', 200, False),
+    ('mixedcase', 'sTaTuS: 410 Gone\r\n', 410, False),
+    ('deniedexit', 'Status: 403 Forbidden\r\n', 403, True),
+    ('redirectexit', 'Status: 307 Temporary Redirect\r\nLocation: /next\r\n', 307, True),
+    ('invalidexit', 'Status: invalid\r\nLocation: /next\r\n', 302, True),
+]
+for fork in (0, 1):
+    # Own every test file and process; cleanup also runs on assertion failures.
+    with tempfile.TemporaryDirectory(prefix='cgi-status-', dir=task) as directory:
+        work = Path(directory)
+        root = work / 'root'
+        root.mkdir()
+        for name, headers, expected, exits in cases:
+            cgi = root / (name + '.cgi')
+            response = headers + 'Content-type: text/plain\r\n\r\n'
+            literal = response.replace('\\', '\\\\').replace('"', '\\"').replace('\r', '\\r').replace('\n', '\\n')
+            cgi.write_text('#!/usr/bin/perl\nprint "' + literal + '";\n'
+                           'print $miniserv::doing_cgi_eval ? "internal" : "forked";\n'
+                           + ('exit(0);\n' if exits else ''))
+            cgi.chmod(0o700)
+        # Ask the guest for an available port instead of touching an existing server.
+        with socket.socket() as sock:
+            sock.bind(('127.0.0.1', 0))
+            port = sock.getsockname()[1]
+        config = dict(port=port, bind='127.0.0.1', root=root, ssl=0,
+                      pidfile=work/'miniserv.pid', logfile=work/'requests.log',
+                      errorlog=work/'errors.log', session=0, passdelay=0,
+                      log=1, logclear=0, syslog=0, addtype_cgi='internal/cgi',
+                      unauthcgi='^/', forkcgis=fork, internalcgis=0)
+        conf = work / 'miniserv.conf'
+        conf.write_text(''.join(f'{key}={value}\n' for key, value in config.items()))
+        server = subprocess.Popen(['perl', str(source/'miniserv.pl'), '--nofork', str(conf)],
+                                  stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
+                                  stderr=subprocess.DEVNULL)
+        try:
+            # Wait for a real HTTP response before testing, with a bounded deadline.
+            for attempt in range(100):
+                if server.poll() is not None:
+                    raise RuntimeError('Fixture server stopped during startup')
+                try:
+                    if request(port, 'normal.cgi')[0] == 200:
+                        break
+                except OSError:
+                    pass
+                time.sleep(.05)
+            else:
+                raise RuntimeError('Fixture server did not become ready')
+            for name, headers, expected, exits in cases:
+                status, received, body = request(port, name + '.cgi')
+                label = f'forkcgis={fork} {name}'
+                check(status == expected, f'{label}: HTTP {status}, expected {expected}')
+                check(not any(key.lower() == 'status' for key, value in received),
+                      label + ': control header is not forwarded')
+                if name == 'normal':
+                    # Prove both execution modes were used, rather than trusting config.
+                    check(body == ('forked' if fork else 'internal'), label + ': execution mode')
+                # Logging can finish just after the response socket closes.
+                pattern = rf'"GET /{name}\.cgi HTTP/1\.[01]"\s+(\d+)\b'
+                for attempt in range(100):
+                    matches = re.findall(pattern, (work/'requests.log').read_text())
+                    if matches:
+                        break
+                    time.sleep(.02)
+                check(bool(matches) and int(matches[-1]) == expected,
+                      label + ': request log has the response status')
+        finally:
+            server.terminate()
+            try:
+                server.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                # Kill only the test process if it does not honor termination.
+                server.kill()
+                server.wait(timeout=5)
+print('All CGI status and logging checks passed; fixture servers removed.')

t/miniserv-cgi-status.t

@@ -0,0 +1,44 @@
+#!/usr/bin/perl
+# Test CGI status headers without starting a server or loading Webmin config.
+use strict;
+use warnings;
+no warnings qw(once redefine uninitialized);
+use Test::More;
+use FindBin;
+require "$FindBin::Bin/../miniserv-lib.pl";
+
+# response(headers): capture the real in-process CGI HTTP response writer.
+sub response {
+    my ($headers) = @_;
+    my $output = '';
+    local *miniserv::write_data = sub { $output .= join('', @_); };
+    local *miniserv::write_keep_alive = sub {};
+    local *miniserv::reset_byte_count = sub {};
+    local *miniserv::server_info = sub { return 'fixture'; };
+    local %miniserv::cgiheader;
+    local @miniserv::cgiheader;
+    local $miniserv::headers = '';
+    local $miniserv::doneheaders;
+    local $miniserv::ok_code = 200;
+    local $miniserv::ok_message = 'Document follows';
+    miniserv::write_to_sock($headers."\r\nbody");
+    return $output;
+}
+my $denial = response("Status: 403 Forbidden\r\nContent-type: text/plain\r\n");
+like($denial, qr/^HTTP\/1\.0 403 Forbidden\r\n/, 'CGI denial sets the HTTP status');
+unlike($denial, qr/^Status:/m, 'CGI Status is not forwarded as an HTTP header');
+like(response("Location: /next\r\n"), qr/^HTTP\/1\.0 302 /,
+     'ordinary redirects keep their default status');
+like(response("Status: 307 Temporary Redirect\r\nLocation: /next\r\n"),
+     qr/^HTTP\/1\.0 307 /, 'explicit redirect status is respected');
+like(response("Status: invalid\r\nLocation: /next\r\n"), qr/^HTTP\/1\.0 302 /,
+     'invalid CGI status does not suppress the default redirect');
+
+# Ordinary pages retain the default status; other valid CGI codes also work.
+like(response("Content-type: text/plain\r\n"), qr/^HTTP\/1\.0 200 /,
+     'ordinary CGI response remains successful');
+like(response("Status: 201 Created\r\nContent-type: text/plain\r\n"),
+     qr/^HTTP\/1\.0 201 Created\r\n/, 'CGI creation response sets its status');
+like(response("Status: invalid\r\nContent-type: text/plain\r\n"),
+     qr/^HTTP\/1\.0 200 /, 'invalid CGI status leaves the default unchanged');
+done_testing();

t/ui-lib-widgets.t

@@ -98,7 +98,7 @@ assert_no_handler_injection(main::ui_tip('<b>x</b>', $xss), 'ui_tip');
 my $tip = main::ui_tip('<b>x</b>', 'A <i>tip</i>');
 like($tip, qr/^<span (?=[^>]*\bclass="ui--span ui_tip")(?=[^>]*\baria-label="A tip")(?=[^>]*\bdata-tooltip[\s>])[^>]*><b>x<\/b><\/span>$/,
      'ui_tip uses the theme tooltip attributes of ui_help');
-unlike($tip, qr/data-ui-tip|tabindex/, 'ui_tip draws no tooltip of its own');
+unlike($tip, qr/data-ui-tip|tabindex/, 'ui_tip needs no JavaScript attributes or extra tab stop');
 assert_no_handler_injection(
 	main::ui_dl([ { 'label' => $xss, 'value' => $xss, 'help' => $xss } ]),
 	'ui_dl hash row');
@@ -680,6 +680,19 @@ like(main::ui_form_columns_table('x.cgi', [ [ 'go', 'Go' ] ], 0, undef, undef,
 		'counter-free long lists retain filtering');
 }
 
+# Row titles remain literal text when converted to shared tooltip attributes.
+{
+	my $title = q{Literal <tag> &amp;lt;example&amp;gt; "quoted".};
+	my $html = main::ui_multi_select_list('tip', [],
+		[ { value => 'one', label => 'One', attrs => { title => $title } } ]);
+	my ($tip) = $html =~ /<span (?=[^>]*\bui_tip\b)[^>]*aria-label="([^"]*)"/;
+	is(decode_attr($tip), q{Literal <tag> &lt;example&gt; "quoted".},
+		'CSS tooltip preserves angle brackets, entities, quotes and punctuation');
+	assert_no_handler_injection(main::ui_multi_select_list('tip_injection', [],
+		[ { value => 'one', label => 'One', attrs => { title => $xss } } ]),
+		'CSS row tooltip');
+}
+
 # Legacy attributes must retain row presentation and checkbox behavior.
 {
 	my $tags = q{DISABLED="false" class="managed" style='font-style:italic' title="Policy &amp; owner" data-policy=managed};
@@ -694,11 +707,14 @@ like(main::ui_form_columns_table('x.cgi', [ [ 'go', 'Go' ] ], 0, undef, undef,
 		'legacy classes combine with the widget and disabled row classes');
 	like($html, qr/<div (?=[^>]*\bui_multi_item\b)(?=[^>]*style="font-style:italic")/,
 		'legacy styles apply to the whole row');
-	like($html, qr/title="Policy &amp; owner"/,
-		'legacy title entities are preserved without double escaping');
+	like($html, qr/<span (?=[^>]*\bui_tip\b)(?=[^>]*aria-label="Policy &amp; owner")/,
+		'legacy title entities are preserved in the shared CSS tooltip');
+	like($html, qr/<input (?=[^>]*value="locked")(?=[^>]*aria-description="Policy &amp; owner")/,
+		'checkbox descriptions remain available to assistive technology');
+	unlike($html, qr/\btitle=/, 'row descriptions do not create duplicate native tooltips');
 	like($html, qr/data-policy="managed"/,
 		'unquoted data attributes survive normalization');
-	unlike($html, qr/<input (?=[^>]*value="open")(?=[^>]*\bdisabled\b)/,
+	unlike($html, qr/<input (?=[^>]*value="open")(?=[^>]*\sdisabled=)/,
 		'disabled inside a quoted attribute does not disable the checkbox');
 	like($html, qr/data-note="a=b > c"/,
 		'quoted attribute values retain spaces, equals and greater-than signs');
@@ -719,7 +735,7 @@ like(main::ui_form_columns_table('x.cgi', [ [ 'go', 'Go' ] ], 0, undef, undef,
 		[ [ 'gone', 'Gone', q{disabled title='Retained'} ] ], [ ]);
 	like($missing, qr/<input (?=[^>]*value="gone")(?=[^>]*\bchecked\b)(?=[^>]*\bdisabled\b)/,
 		'missing selected values retain their legacy disabled attribute');
-	like($missing, qr/title="Retained"/, 'missing values retain row attributes');
+	like($missing, qr/aria-label="Retained"/, 'missing values retain row attributes');
 	my $fixed = main::ui_multi_select_list('fixed', [ ],
 		[ [ 'a', 'A', q{checked name=wrong value=wrong type=radio} ] ]);
 	like($fixed, qr/<input (?=[^>]*type='checkbox')(?=[^>]*name="fixed_item")(?=[^>]*value="a")/,
@@ -843,8 +859,8 @@ like(main::ui_form_columns_table('x.cgi', [ [ 'go', 'Go' ] ], 0, undef, undef,
 		'selected attributes take precedence over available option attributes');
 	like($html, qr/<input (?=[^>]*value="c")(?=[^>]*\bdisabled\b)/,
 		'unselected entries keep their option attributes');
-	like($html, qr/title="Selected"/, 'legacy selected row keeps its tooltip');
-	like($html, qr/title="Retained"/, 'missing legacy selection keeps its tooltip');
+	like($html, qr/aria-label="Selected"/, 'legacy selected row keeps its tooltip');
+	like($html, qr/aria-label="Retained"/, 'missing legacy selection keeps its tooltip');
 	is_deeply(\@values, [ [ 'b', 'B (selected)', q{disabled title="Selected"} ],
 		[ 'a', 'R&amp;D' ], [ 'gone', 'Missing', q{title="Retained"} ] ],
 		'legacy selected entries are not modified');

t/ui-multi-select-browser.mjs

@@ -95,7 +95,7 @@ try {
     assert.equal(await evaluate("value('domains')"), 'parent\nchild');
     assert.equal(await evaluate("box('domains').querySelector('[data-ui-multi-level]').hidden"), true);
     assert.equal(await evaluate("box('domains').querySelector('.ui_multi_count').textContent"), '1 selected');
-    assert.equal(await evaluate("box('domains').querySelector('.ui_chip').textContent"), 'Disabled');
+    assert.equal(await evaluate("box('domains').querySelector('.ui_chip').textContent.trim()"), 'Disabled');
     await evaluate("box('domains').querySelector('[data-ui-multi-action=children]').click()");
     assert.equal(await evaluate("box('domains').querySelector('[data-ui-multi-level]').hidden"), false);
     await evaluate(`const mode = box('domains').querySelector('select'); mode.value = '1'; mode.dispatchEvent(new Event('change', {bubbles:true}));`);
@@ -123,6 +123,40 @@ try {
     assert.equal(await evaluate("value('grouped')"), 'image-0');
     assert.equal(await evaluate("box('domains').querySelector('[data-ui-multi-level]').hidden"), true);
 
+    // Row descriptions use CSS on keyboard focus, without native or JS tips.
+    await evaluate(`box('grouped').scrollIntoView();
+        window.tip = box('grouped').querySelector('.ui_tip');
+        tip.querySelector('input').focus();`);
+    assert.equal(await evaluate("getComputedStyle(tip, '::before').display"), 'block');
+    assert.equal(await evaluate("tip.getAttribute('aria-label')"), 'Description for image 0. Literal <text> & punctuation.');
+    assert.equal(await evaluate("tip.querySelector('input').getAttribute('aria-description')"),
+        await evaluate("tip.getAttribute('aria-label')"));
+    assert.equal(await evaluate("box('grouped').querySelectorAll('.ui_multi_item[title], .tooltip').length"), 0);
+    // The arrow follows the checkbox center, not a fixed label offset.
+    const arrowOffset = await evaluate(`{ const a = getComputedStyle(tip, '::after');
+        const r = tip.querySelector('input').getBoundingClientRect();
+        parseFloat(a.left) + parseFloat(a.width) / 2 - (r.x + r.width / 2); }`);
+    assert.ok(Math.abs(arrowOffset) < 1, 'tooltip arrow points at the checkbox');
+    const nextTip = await evaluate(`{ const r = box('grouped').querySelectorAll('.ui_tip')[1].getBoundingClientRect();
+        ({x: r.x + 5, y: r.y + r.height / 2}); }`);
+    await page('Input.dispatchMouseEvent', { type: 'mouseMoved', ...nextTip });
+    assert.equal(await evaluate("getComputedStyle(tip, '::before').display"), 'none', 'hover takes priority over old focus');
+    assert.equal(await evaluate("getComputedStyle(box('grouped').querySelectorAll('.ui_tip')[1], '::before').display"), 'block');
+    await page('Input.dispatchMouseEvent', { type: 'mouseMoved', x: 1100, y: 0 });
+    const tipColors = [];
+    for (const scheme of ['light', 'dark']) {
+        await evaluate(`box('grouped').dataset.uiScheme = '${scheme}'`);
+        tipColors.push(await evaluate("getComputedStyle(tip, '::before').backgroundImage"));
+    }
+    assert.notEqual(tipColors[0], tipColors[1], 'tooltip colors follow the widget scheme');
+    await page('Emulation.setDeviceMetricsOverride', { width: 375, height: 900, deviceScaleFactor: 1, mobile: false });
+    const tipBounds = await evaluate(`{ const s = getComputedStyle(tip, '::before');
+        ({left: parseFloat(s.left), right: parseFloat(s.left) + parseFloat(s.width)}); }`);
+    assert.ok(tipBounds.left >= 0 && tipBounds.right <= 375, 'tooltip fits the narrow viewport');
+    await evaluate("tip.querySelector('input').blur()");
+    assert.equal(await evaluate("getComputedStyle(tip, '::before').display"), 'none');
+    await page('Emulation.setDeviceMetricsOverride', { width: 1120, height: 900, deviceScaleFactor: 1, mobile: false });
+
     // Inspect real Cloudmin metadata in both schemes and a narrow viewport.
     if (process.argv[3]) {
         await evaluate("document.querySelector('#checks').hidden = true");

t/ui-multi-select-fixture.pl

@@ -22,6 +22,7 @@ my @options = map { { value => "image-$_", label => "Image $_",
 	group => { label => $_ < 4 ? 'Recommended' : $_ < 8 ? 'Supported' : 'Older',
 		state => $_ < 4 ? 'success' : $_ < 8 ? 'neutral' : 'warning' },
 	disabled => $_ == 9,
+	attrs => { title => "Description for image $_. Literal <text> & punctuation." },
 	metadata => [ { label => '10 GiB', title => 'Minimum disk size: 10 GiB', icon => 'hard-drive' },
 		{ label => $_ < 8 ? 'Support until 2030' : 'Support ended 2025',
 			icon => 'clock', state => $_ < 8 ? 'neutral' : 'warning' } ] }

t/web-lib-funcs-encoding.t

@@ -173,6 +173,20 @@ subtest 'Data::Dumper format' => sub {
 	require Data::Dumper;
 	require JSON::PP;
 
+	# Newer peers emit native booleans as !!1 and !!0, including RPC results.
+	for my $value (0, 1) {
+		is(main::unserialise_variable("\$VAR1 = !!$value;"), !!$value,
+		   "literal !!$value retains Perl boolean semantics");
+		is_deeply(main::unserialise_variable(
+			"\$VAR1 = {'rv' => !!$value, 'status' => 1};"),
+			{ 'rv' => !!$value, 'status' => 1 },
+			"fast RPC response accepts !!$value");
+		}
+	is_deeply(main::unserialise_variable(q{$VAR1 = [!!1, !!0, \!!0];}),
+		[ !!1, !!0, \!!0 ], 'booleans work in arrays and scalar references');
+	is_deeply(main::unserialise_variable(q{$VAR1 = ['!!1', "!!0"];}),
+		[ '!!1', '!!0' ], 'quoted boolean text remains unchanged');
+
 	# Spot-check the supported scalar, reference, object and container forms.
 	my $str = 'x';
 	my $shared = [ 1 ];
@@ -182,7 +196,7 @@ subtest 'Data::Dumper format' => sub {
 		  'sref' => \$str, 'rref' => \\'y', 'a' => $shared, 'b' => $shared,
 		  'obj' => bless({ 'x' => 1 }, 'RPC::Untrusted'),
 		  'sobj' => bless(\$str, 'RPC::Untrusted'),
-		  'empty' => [], 'eh' => {} };
+		  'empty' => [], 'eh' => {}, 'true' => !!1, 'false' => !!0 };
 	$v->{'self'} = $v;
 	# Data::Dumper cannot preserve a reference to a hash value if it sees that
 	# reference first. Sorting keys makes this test deterministic.
@@ -263,6 +277,15 @@ subtest 'Data::Dumper format' => sub {
 	# Reject hostile input without executing it.
 	our $ran = 0;
 	my @hostile = (
+		q{$VAR1 = !!2;},
+		q{$VAR1 = !!10;},
+		q{$VAR1 = !!01;},
+		q{$VAR1 = !!1e0;},
+		q{$VAR1 = !!0.0;},
+		q{$VAR1 = !!1foo;},
+		q{$VAR1 = !!($main::ran = 1);},
+		q{$VAR1 = !!0 || do { $main::ran = 1 };},
+		q{$VAR1 = !!1; $main::ran = 1;},
 		q{$VAR1 = {}; $main::ran = 1;},
 		q{$VAR1 = "@{[ $main::ran = 1 ]}";},
 		q{$VAR1 = "${\ ($main::ran = 1)}";},

t/web-lib-funcs-unlink.t

@@ -0,0 +1,97 @@
+#!/usr/bin/perl
+# Exercise unlink_file and cached reads using only temporary fixture files.
+use strict;
+use warnings;
+no warnings 'once';
+use Test::More;
+use File::Temp qw(tempdir);
+use FindBin;
+require "$FindBin::Bin/../web-lib-funcs.pl";
+
+# Avoid loading workstation configuration or writing Webmin debug logs.
+local %main::gconfig = ();
+local $main::readonly_mode_cache = 0;
+local @main::filename_callbacks = ();
+my $root = tempdir(CLEANUP => 1);
+
+# write_fixture(file, contents): write directly without updating Webmin caches.
+sub write_fixture
+{
+my ($file, $contents) = @_;
+open(my $fh, '>', $file) or die "Create fixture $file: $!";
+print $fh $contents or die "Write fixture $file: $!";
+close($fh) or die "Close fixture $file: $!";
+}
+
+# Both readers and deletion must use the translated path as their cache key.
+foreach my $translated (0, 1) {
+    foreach my $recreate (0, 1) {
+        my $label = ($translated ? 'translated' : 'direct').
+            ($recreate ? ' recreated' : ' deleted');
+        my $file = "$root/$translated-$recreate.conf";
+        my $requested = $translated ? "$root/alias.conf" : $file;
+        # Map the logical name to a real file entirely within the fixture tree.
+        local @main::filename_callbacks = ([undef, sub {
+            return $_[0] eq $requested ? $file : $_[0];
+        }, []]);
+        write_fixture($file, "value=old\nremoved=1\n");
+        my %before;
+        ok(main::read_file_cached($requested, \%before), "$label: initial read");
+        is($before{value}, 'old', "$label: initial contents");
+        ok(main::unlink_file($requested), "$label: deletion succeeds");
+        ok(!-e $file, "$label: file is gone");
+        my %after;
+        if ($recreate) {
+            # A new file at the same path must not inherit the old cached keys.
+            write_fixture($file, "value=new\n");
+            ok(main::read_file_cached($requested, \%after), "$label: new read");
+            is_deeply(\%after, {value => 'new'}, "$label: only new contents");
+        }
+        else {
+            # A successful cached read must not survive removal of its file.
+            ok(!main::read_file_cached($requested, \%after), "$label: read fails");
+            is_deeply(\%after, {}, "$label: no deleted contents returned");
+        }
+    }
+}
+
+# External writers can create a file after Webmin has cached its absence.
+my $missing = "$root/missing.conf";
+my %data;
+ok(!main::read_file_cached($missing, \%data), 'cache a missing file');
+write_fixture($missing, "value=temporary\n");
+ok(main::unlink_file($missing), 'delete a file previously cached as missing');
+write_fixture($missing, "value=recreated\n");
+ok(main::read_file_cached($missing, \%data), 'recreated file is readable');
+is_deeply(\%data, {value => 'recreated'}, 'old missing-file flag is cleared');
+
+# A failed deletion must still report its error and process later arguments.
+my @files = map { "$root/batch-$_.conf" } (1, 2);
+foreach my $file (@files) {
+    write_fixture($file, "value=cached\n");
+    my %cached;
+    ok(main::read_file_cached($file, \%cached), 'cache a batch file');
+}
+my ($ok, $err) = main::unlink_file("$root/absent.conf", @files);
+ok(!$ok, 'batch reports a failed deletion');
+ok($err, 'batch returns the deletion error');
+foreach my $file (@files) {
+    my %after;
+    ok(!-e $file, 'batch still deletes later files');
+    ok(!main::read_file_cached($file, \%after), 'later file cache is invalidated');
+}
+
+# Read-only mode must leave both the file and its cached contents available.
+my $readonly = "$root/readonly.conf";
+write_fixture($readonly, "value=retained\n");
+my %before;
+ok(main::read_file_cached($readonly, \%before), 'cache the read-only fixture');
+{
+    local $main::readonly_mode_cache = 1;
+    ok(main::unlink_file($readonly), 'read-only call retains its success return');
+}
+ok(-e $readonly, 'read-only call does not delete the file');
+my %after;
+ok(main::read_file_cached($readonly, \%after), 'read-only file remains readable');
+is_deeply(\%after, \%before, 'read-only contents remain unchanged');
+done_testing();

ui-lib.pl

@@ -4530,9 +4530,10 @@ return &ui_tag('code', &html_escape($str), { 'class' => 'ui_code' });
 
 =head2 ui_tip(content-html, tip)
 
-Wraps arbitrary HTML content so the given plain-text tip is shown on
-hover by the theme's tooltip, the same one ui_help uses. For a plain
-question-mark help bubble, use ui_help instead.
+Wraps HTML content in a CSS tooltip, shown on hover or focus within the
+content. Load ui_page_assets when using this outside a widget page.
+The tip is text; HTML tags are stripped. For a question-mark help bubble,
+use ui_help instead.
 
 =cut
 sub ui_tip
@@ -6132,7 +6133,7 @@ Size, add-if-missing, titles and width are ignored.
 
 =item values - Array reference of selected scalars or [ value, label, attributes ] entries. Attributes are also retained when a selected entry is missing from options.
 
-=item options - Array reference of [ value, label, attributes ] entries or hashes with keys value, label, suffix (muted text after the label), level (indentation depth), tag (chip at the right), disabled and attrs. Attributes may be a hash or trusted HTML attribute string, as in ui_select. disabled applies to the checkbox; other attributes, such as title, style and class, apply to the row. Widget identity and selection remain controlled by the library.
+=item options - Array reference of [ value, label, attributes ] entries or hashes with keys value, label, suffix (muted text after the label), level (indentation depth), tag (chip at the right), disabled and attrs. Attributes may be a hash or trusted HTML attribute string, as in ui_select. disabled applies to the checkbox; title becomes a CSS tooltip around the checkbox label; style and class apply to the row. Widget identity and selection remain controlled by the library.
 
 =item badges - An option hash may also contain a badges array. Each entry holds ui_badge arguments: [ label, state, options ]. Badges appear separately at the right, beside any tag. Their labels are escaped and included in filtering.
 
@@ -6402,10 +6403,16 @@ foreach my $it (@items) {
 	# Checkbox renderers may place HTML outside their label element.
 	my $ctags = "data-ui-multi-item='1'";
 	$ctags .= ' aria-label="'.$filter_label.'"' if ($opts->{'html'});
+	# Reuse the CSS tooltip for row titles, without a second native tooltip.
+	my $tip = delete($it->{'attrs'}->{'title'});
+	$ctags .= ' aria-description="'.&html_escape($tip, 1).'"'
+		if (defined($tip) && $tip ne '');
 	my $row = &ui_checkbox($inputname.'_item', $val, $label,
 			       $selected{$val} ? 1 : 0,
 			       $ctags,
 			       $dis || $it->{'disabled'} ? 1 : 0);
+	$row = &ui_tip($row, &html_escape($tip, 1))
+		if (defined($tip) && $tip ne '');
 	if ($note && $it->{'kids'}) {
 		my $nattrs = { 'class' => 'ui_multi_note' };
 		$nattrs->{'hidden'} = undef if (!$folded);
@@ -6415,9 +6422,9 @@ foreach my $it (@items) {
 	# Keep plain tags compatible while allowing separate state badges.
 	my @badges = ref($it->{'badges'}) eq 'ARRAY' ?
 		grep { ref($_) eq 'ARRAY' && defined($_->[0]) } @{$it->{'badges'}} : ();
-	my $side = defined($it->{'tag'}) && $it->{'tag'} ne '' ?
-		&ui_chip($it->{'tag'}) : '';
-	$side .= join('', map { &ui_badge(@$_) } @badges);
+	# Badges and metadata come first, so the tag chip stays a fixed
+	# right-hand column across rows with and without badges.
+	my $side = join('', map { &ui_badge(@$_) } @badges);
 	# Several facts share one badge, with plain labels and accessible tooltips.
 	my @metadata = ref($it->{'metadata'}) eq 'ARRAY' ?
 		grep { ref($_) eq 'HASH' && defined($_->{'label'}) &&
@@ -6435,6 +6442,8 @@ foreach my $it (@items) {
 			{ 'class' => 'ui_badge ui_badge_neutral ui_badge_small '.
 				'ui_badge_rounded ui_multi_metadata' });
 		}
+	$side .= &ui_chip($it->{'tag'})
+		if (defined($it->{'tag'}) && $it->{'tag'} ne '');
 	$row .= &ui_tag('span', $side,
 		{ 'class' => &_ui_class('ui_multi_side', @badges ? 'ui_multi_badges' : undef) })
 		if (length($side));

unauthenticated/css/ui-lib.css

@@ -124,6 +124,144 @@
 	text-align: left;
 }
 
+/* Shared CSS tooltips. Theme tokens keep the bubble and its arrow readable
+ * in light and dark palettes. Ordinary help bubbles retain their theme style. */
+.ui_tip[data-tooltip][aria-label] {
+	position: relative;
+	display: inline-block;
+	vertical-align: baseline;
+	--ui-tip-bg: var(--ui-fg, var(--text-color, #212121));
+	--ui-tip-fg: var(--ui-surface, var(--main-content, var(--bg-color, #fff)));
+}
+.ui_tip[data-tooltip][aria-label]::before,
+.ui_tip[data-tooltip][aria-label]::after {
+	position: absolute;
+	pointer-events: none;
+	display: none;
+	z-index: 1060;
+}
+.ui_tip[data-tooltip][aria-label]::before {
+	content: attr(aria-label);
+	bottom: calc(100% + 8px);
+	left: 0;
+	width: max-content;
+	box-sizing: border-box;
+	max-width: min(32rem, calc(100vw - 24px));
+	padding: 7px 10px;
+	border-radius: 5px;
+	/* Theme text colors may have alpha; paint over a solid surface. */
+	background: linear-gradient(var(--ui-tip-bg), var(--ui-tip-bg)),
+		var(--ui-tip-fg);
+	color: var(--ui-tip-fg);
+	font-size: 13px;
+	line-height: 1.45;
+	font-weight: normal;
+	font-style: normal;
+	text-align: left;
+	text-transform: none;
+	text-shadow: none;
+	white-space: normal;
+	overflow-wrap: anywhere;
+}
+.ui_tip[data-tooltip][aria-label]::after {
+	content: '';
+	bottom: calc(100% + 4px);
+	left: 12px;
+	width: 8px;
+	height: 8px;
+	background: linear-gradient(var(--ui-tip-bg), var(--ui-tip-bg)),
+		var(--ui-tip-fg);
+	transform: rotate(45deg);
+}
+.ui_tip[data-tooltip][aria-label]:is(:hover, :focus-within)::before,
+.ui_tip[data-tooltip][aria-label]:is(:hover, :focus-within)::after {
+	display: block;
+}
+/* A hovered row takes priority over the checkbox that still has focus. */
+.ui_multi:has(.ui_tip:hover) .ui_tip:not(:hover)::before,
+.ui_multi:has(.ui_tip:hover) .ui_tip:not(:hover)::after {
+	display: none;
+}
+/* Fixed anchors escape scrolling lists. Scope each pair so repeated entries
+ * cannot attach to another row's tooltip. Older browsers use the absolute
+ * positioning above. */
+@supports (anchor-scope: --ui-tip) {
+	.ui_tip[data-tooltip][aria-label] {
+		anchor-name: --ui-tip;
+		anchor-scope: --ui-tip, --ui-tip-box, --ui-tip-input;
+	}
+	.ui_tip[data-tooltip][aria-label]::before {
+		position: fixed;
+		position-anchor: --ui-tip;
+		anchor-name: --ui-tip-box;
+		bottom: anchor(top);
+		left: anchor(left);
+		margin-bottom: 8px;
+		position-try-fallbacks: --ui-tip-right, --ui-tip-fit,
+			--ui-tip-below, --ui-tip-below-right, --ui-tip-below-fit;
+		position-visibility: anchors-visible;
+	}
+	.ui_tip[data-tooltip][aria-label]::after {
+		position: fixed;
+		position-anchor: --ui-tip;
+		bottom: auto;
+		/* Attach the arrow to the nearest bubble edge after a position change. */
+		top: calc(clamp(anchor(--ui-tip-box top), anchor(bottom),
+			anchor(--ui-tip-box bottom)) - 4px);
+		left: clamp(calc(anchor(--ui-tip-box left) + 8px),
+			calc(anchor(left) + 12px), calc(anchor(--ui-tip-box right) - 16px));
+		position-visibility: anchors-visible;
+	}
+	/* Point list descriptions at the checkbox, whose size varies by theme.
+	 * Leave room beside it for the bubble's rounded corner. */
+	.ui_multi .ui_tip input[type="checkbox"] {
+		anchor-name: --ui-tip-input;
+	}
+	.ui_multi .ui_tip[data-tooltip][aria-label]::before {
+		left: calc(anchor(--ui-tip-input center) - 12px);
+	}
+	.ui_multi .ui_tip[data-tooltip][aria-label]::after {
+		left: clamp(calc(anchor(--ui-tip-box left) + 8px),
+			calc(anchor(--ui-tip-input center) - 4px),
+			calc(anchor(--ui-tip-box right) - 16px));
+	}
+	@position-try --ui-tip-right {
+		left: auto;
+		right: anchor(right);
+	}
+	/* A long description can need more room than either side of its label. */
+	@position-try --ui-tip-fit {
+		left: 12px;
+		right: 12px;
+		width: auto;
+	}
+	@position-try --ui-tip-below {
+		bottom: auto;
+		top: anchor(bottom);
+		margin-top: 8px;
+		margin-bottom: 0;
+	}
+	@position-try --ui-tip-below-right {
+		bottom: auto;
+		top: anchor(bottom);
+		left: auto;
+		right: anchor(right);
+		margin-top: 8px;
+		margin-bottom: 0;
+	}
+	@position-try --ui-tip-below-fit {
+		bottom: auto;
+		top: anchor(bottom);
+		left: 12px;
+		right: 12px;
+		width: auto;
+		margin-top: 8px;
+		margin-bottom: 0;
+	}
+}
+.ui_multi .ui_tip { display: inline-flex; align-items: center; min-width: 0; }
+.ui_multi .ui_tip > input[type="checkbox"] { margin-right: 6px; }
+
 /* Built-in dark palette, enabled with data-ui-scheme="dark" on any
  * ancestor element (or on the .ui_page wrapper itself, via the scheme
  * option of ui_page_start). With data-ui-scheme="auto" the dark
@@ -281,7 +419,9 @@ body:has(.ui_page[data-ui-scheme="dark"]) {
 .ui_field, .ui_field * {
 	box-sizing: border-box;
 }
-.ui_page h1, .ui_page h2, .ui_page p, .ui_page dl, .ui_page dd { margin: 0; }
+/* Reset widget paragraphs without removing spacing from help and other prose. */
+.ui_page h1, .ui_page h2, .ui_page p.ui--p,
+.ui_page dl, .ui_page dd { margin: 0; }
 .ui_page svg.ui_svg_icon { flex-shrink: 0; vertical-align: -0.18em; }
 
 /* Underline plain text links, but keep links styled as buttons undecorated. */
@@ -1595,23 +1735,24 @@ a.ui_list_link:hover { color: var(--ui-accent) !important; }
 	white-space: nowrap;
 }
 .ui_multi_side { margin-left: auto; padding-left: 8px; flex-shrink: 0; }
-/* Metadata stays secondary to the label, with room between rows and badges. */
-.ui_multi_item:has(> .ui_multi_badges) { padding: 7px 10px; }
+/* Badges share the chip's size and the row's normal padding, so rows with
+ * badges line up with rows without them. */
 .ui_multi_badges {
 	display: flex;
 	flex-wrap: wrap;
 	align-items: center;
 	justify-content: flex-end;
-	gap: 6px 8px;
+	gap: 6px;
 	flex-shrink: 1;
-	padding-left: 16px;
 }
 .ui_multi_badges .ui_badge {
-	padding: 3px 10px;
+	padding: 1px 8px;
+	font-size: 0.86em;
 	font-weight: 400;
-	line-height: 1.4;
+	line-height: 1.5;
 	border-color: color-mix(in srgb, currentColor 16%, transparent);
 }
+.ui_multi_badges .ui_badge .ui_svg_icon { width: 1em; height: 1em; }
 .ui_multi_badges .ui_badge_neutral {
 	background: color-mix(in srgb, var(--ui-neutral-soft) 35%, transparent);
 	color: var(--ui-fg-muted);

vendor_perl/Data/Dumper/Parser.pm

@@ -56,6 +56,10 @@ if ($$s =~ /\G'/gc) {
 elsif ($$s =~ /\G"/gc) {
 	return _parse_double($s);
 	}
+elsif ($$s =~ /\G!!([01])(?![A-Za-z0-9_.])/gc) {
+	# Read native boolean literals without evaluating arbitrary expressions.
+	return $1 eq '1';
+	}
 elsif ($$s =~ /\G(-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+)?)(?![A-Za-z0-9_.])/gc) {
 	# Return an unquoted number as a numeric scalar.
 	return $1 + 0;

web-lib-funcs.pl

@@ -1235,8 +1235,7 @@ while(1) {
 			$count_lines = 0;
 			}
 		if ($max && $got > $max) {
-			#print STDERR "over limit of $max\n";
-			#&error($err);
+			&error($err);
 			}
 		if (!$line) {
 			# Unexpected EOF?
@@ -8586,6 +8585,13 @@ foreach my $f (@_) {
 			$rv = 0;
 			$err = $!;
 			}
+		else {
+			# Cached contents must not outlive the file, or a later
+			# read_file_cached call in this process returns stale
+			# data.
+			delete($main::read_file_cache{$realf});
+			delete($main::read_file_missing{$realf});
+			}
 		}
 	}
 return wantarray ? ($rv, $err) : $rv;

webmin/edit_session.cgi

@@ -133,7 +133,8 @@ print &ui_table_row($text{'session_pam'},
 
 print &ui_table_row($text{'session_popts'},
 	ui_checkbox("pam_conv", 1, $text{'session_pamconv'},
-		     $miniserv{'pam_conv'}).
+		     $miniserv{'pam_conv'}.
+		     ($miniserv{'twofactor_provider'} ? " $text{'session_pamwarn'}" : "")).
 	"<br>".
 	ui_checkbox("pam_end", 1, $text{'session_pamend'},
 		     $miniserv{'pam_end'}).

webmin/lang/en

@@ -664,6 +664,7 @@ session_pamoff=Never use PAM for Unix authentication
 session_pfile=If PAM is unavailable or disabled, read users and passwords from file $1 columns $2 and $3
 session_popts=Password options
 session_pamconv=Support full PAM conversations?
+session_pamwarn=(WARNING! Bypasses 2FA authentication!)
 session_pamend=Pass on PAM status to other modules?
 session_cmddef=Expired password change
 session_cmddef1=Change expired passwords via PAM