From a4f7cb1f8c3eed89201283cf6fac9e450a5fceab Mon Sep 17 00:00:00 2001 From: iliajie Date: Wed, 17 May 2023 16:59:00 +0300 Subject: [PATCH 01/15] Add API to check if given URL redirects somewhere --- WebminCore.pm | 2 +- web-lib-funcs.pl | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/WebminCore.pm b/WebminCore.pm index 0e22ddd86..35fdbdce2 100644 --- a/WebminCore.pm +++ b/WebminCore.pm @@ -23,7 +23,7 @@ $main::export_to_caller = 1; # Add functions in web-lib-funcs.pl # Generated with : # grep -h "^sub " web-lib-funcs.pl ui-lib.pl | sed -e 's/sub //' | xargs echo -@EXPORT = qw(read_file read_file_cached read_file_cached_with_stat write_file html_escape html_strip quote_escape quote_javascript tempname_dir tempname transname transname_timestamped trunc indexof indexoflc sysprint check_ipaddress check_ip6address generate_icon urlize un_urlize include copydata ReadParseMime ReadParse read_fully read_parse_mime_callback read_parse_mime_javascript PrintHeader header get_html_title get_html_framed_title get_html_status_line popup_header footer popup_footer load_module_preferences load_theme_library redirect kill_byname kill_byname_logged find_byname error popup_error register_error_handler call_error_handlers error_setup wait_for fast_wait_for has_command make_date file_chooser_button popup_window_button popup_window_link read_acl acl_filename acl_check get_miniserv_config_file get_miniserv_config put_miniserv_config restart_miniserv reload_miniserv check_os_support http_download complete_http_download http_post ftp_download ftp_upload no_proxy open_socket download_timeout ftp_command to_ipaddress to_ip6address to_hostname icons_table replace_meta replace_file_line read_file_lines flush_file_lines unflush_file_lines unix_user_input unix_group_input hlink user_chooser_button group_chooser_button foreign_check foreign_exists foreign_available foreign_require foreign_call foreign_config foreign_installed foreign_defined get_system_hostname get_webmin_version get_webmin_version_release get_webmin_full_version get_module_acl get_group_module_acl save_module_acl save_group_module_acl init_config load_language_auto load_language text_subs text encode_base64 decode_base64 encode_base32 decode_base32 get_module_info get_all_module_infos list_themes get_theme_info list_locales list_languages safe_language read_env_file write_env_file lock_file unlock_file test_lock unlock_all_files can_lock_file webmin_log additional_log var_dump webmin_debug_log system_logged backquote_logged backquote_with_timeout backquote_command kill_logged rename_logged rename_file symlink_logged symlink_file link_file make_dir make_dir_recursive set_ownership_permissions unlink_logged unlink_file copy_source_dest move_source_dest remote_session_name remote_foreign_require remote_foreign_call remote_foreign_check remote_foreign_config remote_eval remote_write remote_read remote_finished remote_error_setup remote_rpc_call remote_multi_callback remote_multi_callback_error serialise_variable unserialise_variable other_groups date_chooser_button help_file read_help_file seed_random disk_usage_kb recursive_disk_usage help_search_link make_http_connection validate_ssl_connection read_http_connection write_http_connection close_http_connection clean_environment reset_environment clean_language progress_callback switch_to_remote_user switch_to_unix_user eval_as_unix_user create_user_config_dirs create_missing_homedir filter_javascript resolve_links simplify_path same_file flush_webmin_caches list_usermods available_usermods get_available_module_infos get_visible_module_infos get_visible_modules_categories is_under_directory parse_http_url check_clicks_function load_entities_map entities_to_ascii get_product_name get_charset get_display_hostname save_module_config save_user_module_config nice_size get_perl_path get_goto_module select_all_link select_invert_link select_rows_link check_pid_file get_mod_lib module_root_directory list_mime_types guess_mime_type open_tempfile close_tempfile print_tempfile is_selinux_enabled get_clear_file_attributes reset_file_attributes cleanup_tempnames open_lock_tempfile END month_to_number number_to_month get_rbac_module_acl supports_rbac supports_ipv6 use_rbac_module_acl execute_command open_readfile open_execute_command translate_filename translate_command register_filename_callback register_command_callback capture_function_output capture_function_output_tempfile modules_chooser_button substitute_template substitute_pattern running_in_zone running_in_vserver running_in_xen running_in_openvz list_categories is_readonly_mode command_as_user list_osdn_mirrors convert_osdn_url get_current_dir supports_users supports_symlinks quote_path get_windows_root read_file_contents write_file_contents read_file_contents_limit unix_crypt split_quoted_string write_to_http_cache check_in_http_cache supports_javascript get_module_name get_module_variable clear_time_locale reset_time_locale callers_package web_libs_package get_userdb_string connect_userdb disconnect_userdb split_userdb_string uniquelc list_combined_webmin_menu list_modules_webmin_menu module_to_menu_item list_combined_system_info shell_is_bash compare_version_numbers convert_to_json convert_from_json print_json get_referer_relative get_webmin_email_url get_webmin_browser_url trim ui_link ui_help ui_img ui_link_button ui_table_start ui_table_end ui_table_row ui_table_hr ui_table_span ui_columns_start ui_columns_row ui_columns_header ui_checked_columns_row ui_radio_columns_row ui_columns_end ui_columns_table ui_form_columns_table ui_form_elements_wrapper ui_form_start ui_form_end ui_textbox ui_filebox ui_bytesbox ui_upload ui_password ui_hidden ui_select ui_multi_select ui_multi_select_javascript ui_radio ui_yesno_radio ui_checkbox ui_oneradio ui_textarea ui_user_textbox ui_users_textbox ui_group_textbox ui_groups_textbox ui_opt_textbox ui_submit ui_reset ui_button ui_date_input ui_buttons_start ui_buttons_end ui_buttons_row ui_buttons_hr ui_post_header ui_pre_footer ui_print_header ui_print_unbuffered_header ui_print_footer ui_config_link ui_print_endpage ui_subheading ui_links_row ui_hidden_javascript ui_hidden_start ui_hidden_end ui_hidden_table_row_start ui_hidden_table_row_end ui_hidden_table_start ui_hidden_table_end ui_tabs_start ui_tabs_end ui_tabs_start_tab ui_tabs_start_tabletab ui_tabs_end_tab ui_tabs_end_tabletab ui_max_text_width ui_radio_selector ui_radio_selector_javascript ui_grid_table ui_radio_table ui_up_down_arrows ui_hr ui_nav_link ui_confirmation_form ui_text_color ui_alert_box js_disable_inputs ui_page_flipper js_checkbox_disable js_redirect ui_webmin_link ui_line_break_double ui_page_refresh ui_details ui_div_row ui_paginations ui_hide_outside_of_viewport ui_read_file_contents_limit get_python_cmd get_buffer_size get_buffer_size_binary get_webprefix get_sub_ref_name setvar getvar delvar print_call_stack webmin_user_can_rpc webmin_user_login_mode webmin_user_is_admin webmin_user_is get_current_theme_info_cached miniserv_using_default_cert is_int float is_float parse_accepted_language get_default_system_locale get_http_cookie create_wrapper); +@EXPORT = qw(read_file read_file_cached read_file_cached_with_stat write_file html_escape html_strip quote_escape quote_javascript tempname_dir tempname transname transname_timestamped trunc indexof indexoflc sysprint check_ipaddress check_ip6address generate_icon urlize un_urlize include copydata ReadParseMime ReadParse read_fully read_parse_mime_callback read_parse_mime_javascript PrintHeader header get_html_title get_html_framed_title get_html_status_line popup_header footer popup_footer load_module_preferences load_theme_library redirect kill_byname kill_byname_logged find_byname error popup_error register_error_handler call_error_handlers error_setup wait_for fast_wait_for has_command make_date file_chooser_button popup_window_button popup_window_link read_acl acl_filename acl_check get_miniserv_config_file get_miniserv_config put_miniserv_config restart_miniserv reload_miniserv check_os_support http_download complete_http_download http_post ftp_download ftp_upload no_proxy open_socket download_timeout ftp_command to_ipaddress to_ip6address to_hostname icons_table replace_meta replace_file_line read_file_lines flush_file_lines unflush_file_lines unix_user_input unix_group_input hlink user_chooser_button group_chooser_button foreign_check foreign_exists foreign_available foreign_require foreign_call foreign_config foreign_installed foreign_defined get_system_hostname get_webmin_version get_webmin_version_release get_webmin_full_version get_module_acl get_group_module_acl save_module_acl save_group_module_acl init_config load_language_auto load_language text_subs text encode_base64 decode_base64 encode_base32 decode_base32 get_module_info get_all_module_infos list_themes get_theme_info list_locales list_languages safe_language read_env_file write_env_file lock_file unlock_file test_lock unlock_all_files can_lock_file webmin_log additional_log var_dump webmin_debug_log system_logged backquote_logged backquote_with_timeout backquote_command kill_logged rename_logged rename_file symlink_logged symlink_file link_file make_dir make_dir_recursive set_ownership_permissions unlink_logged unlink_file copy_source_dest move_source_dest remote_session_name remote_foreign_require remote_foreign_call remote_foreign_check remote_foreign_config remote_eval remote_write remote_read remote_finished remote_error_setup remote_rpc_call remote_multi_callback remote_multi_callback_error serialise_variable unserialise_variable other_groups date_chooser_button help_file read_help_file seed_random disk_usage_kb recursive_disk_usage help_search_link make_http_connection validate_ssl_connection read_http_connection write_http_connection close_http_connection clean_environment reset_environment clean_language progress_callback switch_to_remote_user switch_to_unix_user eval_as_unix_user create_user_config_dirs create_missing_homedir filter_javascript resolve_links simplify_path same_file flush_webmin_caches list_usermods available_usermods get_available_module_infos get_visible_module_infos get_visible_modules_categories is_under_directory parse_http_url check_clicks_function load_entities_map entities_to_ascii get_product_name get_charset get_display_hostname save_module_config save_user_module_config nice_size get_perl_path get_goto_module select_all_link select_invert_link select_rows_link check_pid_file get_mod_lib module_root_directory list_mime_types guess_mime_type open_tempfile close_tempfile print_tempfile is_selinux_enabled get_clear_file_attributes reset_file_attributes cleanup_tempnames open_lock_tempfile END month_to_number number_to_month get_rbac_module_acl supports_rbac supports_ipv6 use_rbac_module_acl execute_command open_readfile open_execute_command translate_filename translate_command register_filename_callback register_command_callback capture_function_output capture_function_output_tempfile modules_chooser_button substitute_template substitute_pattern running_in_zone running_in_vserver running_in_xen running_in_openvz list_categories is_readonly_mode command_as_user list_osdn_mirrors convert_osdn_url get_current_dir supports_users supports_symlinks quote_path get_windows_root read_file_contents write_file_contents read_file_contents_limit unix_crypt split_quoted_string write_to_http_cache check_in_http_cache supports_javascript get_module_name get_module_variable clear_time_locale reset_time_locale callers_package web_libs_package get_userdb_string connect_userdb disconnect_userdb split_userdb_string uniquelc list_combined_webmin_menu list_modules_webmin_menu module_to_menu_item list_combined_system_info shell_is_bash compare_version_numbers convert_to_json convert_from_json print_json get_referer_relative get_webmin_email_url get_webmin_browser_url trim ui_link ui_help ui_img ui_link_button ui_table_start ui_table_end ui_table_row ui_table_hr ui_table_span ui_columns_start ui_columns_row ui_columns_header ui_checked_columns_row ui_radio_columns_row ui_columns_end ui_columns_table ui_form_columns_table ui_form_elements_wrapper ui_form_start ui_form_end ui_textbox ui_filebox ui_bytesbox ui_upload ui_password ui_hidden ui_select ui_multi_select ui_multi_select_javascript ui_radio ui_yesno_radio ui_checkbox ui_oneradio ui_textarea ui_user_textbox ui_users_textbox ui_group_textbox ui_groups_textbox ui_opt_textbox ui_submit ui_reset ui_button ui_date_input ui_buttons_start ui_buttons_end ui_buttons_row ui_buttons_hr ui_post_header ui_pre_footer ui_print_header ui_print_unbuffered_header ui_print_footer ui_config_link ui_print_endpage ui_subheading ui_links_row ui_hidden_javascript ui_hidden_start ui_hidden_end ui_hidden_table_row_start ui_hidden_table_row_end ui_hidden_table_start ui_hidden_table_end ui_tabs_start ui_tabs_end ui_tabs_start_tab ui_tabs_start_tabletab ui_tabs_end_tab ui_tabs_end_tabletab ui_max_text_width ui_radio_selector ui_radio_selector_javascript ui_grid_table ui_radio_table ui_up_down_arrows ui_hr ui_nav_link ui_confirmation_form ui_text_color ui_alert_box js_disable_inputs ui_page_flipper js_checkbox_disable js_redirect ui_webmin_link ui_line_break_double ui_page_refresh ui_details ui_div_row ui_paginations ui_hide_outside_of_viewport ui_read_file_contents_limit get_python_cmd get_buffer_size get_buffer_size_binary get_webprefix get_sub_ref_name setvar getvar delvar print_call_stack webmin_user_can_rpc webmin_user_login_mode webmin_user_is_admin webmin_user_is get_current_theme_info_cached miniserv_using_default_cert is_int float is_float parse_accepted_language get_default_system_locale get_http_redirect get_http_cookie create_wrapper); # Add global variables in web-lib.pl push(@EXPORT, qw(&unique)); diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index d6f0a56a5..1369da1e7 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -12997,6 +12997,86 @@ if (!$@ && $locale_system) { return $locale_def; } +=head2 get_http_redirect(url, [page]) + +Check if given URL redirects somewhere + +The parameters are : + +=item url - Given URL to check if it redirects anywhere, e.g. https://google.com + +=item page - Path in URL, e.g. /about as in https://www.google.com/about + +=cut + +sub get_http_redirect +{ +my ($url, $page) = @_; +my ($out, $error, $con_err); +my ($proto, $host, $uport) = $url =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?/; +my ($ssl, $port) = (0, undef); +$port = '80' if (!$uport); +if ($proto eq 'https') { + $ssl = 1; + $port = '443' if (!$uport); + } + +# Set default port and page +$port ||= $uport; +$page ||= '/'; + +# Build headers +my @headers; +push(@headers, [ "Host", $host ]); +push(@headers, [ "User-agent", "Webmin" ]); +push(@headers, [ "Accept-language", "en" ]); + +# Actually download it +$main::download_timed_out = undef; +local $SIG{ALRM} = \&download_timeout; +$timeout = 30 if (!defined($timeout)); +alarm($timeout) if ($timeout); +my $h = &make_http_connection($host, $port, $ssl, "GET", $page, \@headers); +alarm(0) if ($timeout); +$h = $main::download_timed_out if ($main::download_timed_out); +if (ref($h)) { + &write_http_connection($h, "\r\n"); + } +&complete_http_download($h, undef, \$error, undef, undef, $host, $port, + undef, $ssl, 1, $timeout); +my %rs; +$rs{'url'} = $url; +$rs{'proto'} = $proto; +$rs{'host'} = $host; +$rs{'port'} = $port; +$rs{'page'} = $page; +if (ref($h)) { + $rs{'handle'} = $h + } +else { + $rs{'error'} = $h + } +if (ref($h)) { + + if ($h->{'buffer'} =~ /has\s+moved\s+{'url'} = $rurl; + my ($proto, $host, $uport, $path) = $rurl =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*)/; + my $port = '80' if (!$uport); + if ($proto eq 'https') { + $port = '443' if (!$uport); + } + $port ||= $uport; + $rs{'redir'}->{'proto'} = $proto; + $rs{'redir'}->{'host'} = $host; + $rs{'redir'}->{'port'} = $port; + $rs{'redir'}->{'path'} = $path; + } + } +return \%rs; +} + =head2 get_http_cookie(cookie-name) Returns a cookie value based on its name From b0731f6e357ba5a170787cba215358a21162fc82 Mon Sep 17 00:00:00 2001 From: iliajie Date: Wed, 17 May 2023 17:31:42 +0300 Subject: [PATCH 02/15] Add to use `LWP::UserAgent` if available --- web-lib-funcs.pl | 126 +++++++++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 43 deletions(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 1369da1e7..571a8ac4c 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -12997,7 +12997,7 @@ if (!$@ && $locale_system) { return $locale_def; } -=head2 get_http_redirect(url, [page]) +=head2 get_http_redirect(url, [page], [timeout]) Check if given URL redirects somewhere @@ -13007,11 +13007,13 @@ The parameters are : =item page - Path in URL, e.g. /about as in https://www.google.com/about +=item timeout - Timeout for connections, defaults to 30s + =cut sub get_http_redirect { -my ($url, $page) = @_; +my ($url, $page, $timeout) = @_; my ($out, $error, $con_err); my ($proto, $host, $uport) = $url =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?/; my ($ssl, $port) = (0, undef); @@ -13021,57 +13023,95 @@ if ($proto eq 'https') { $port = '443' if (!$uport); } -# Set default port and page +# Set default port, page and timeout $port ||= $uport; $page ||= '/'; +$timeout ||= 30; -# Build headers -my @headers; -push(@headers, [ "Host", $host ]); -push(@headers, [ "User-agent", "Webmin" ]); -push(@headers, [ "Accept-language", "en" ]); - -# Actually download it -$main::download_timed_out = undef; -local $SIG{ALRM} = \&download_timeout; -$timeout = 30 if (!defined($timeout)); -alarm($timeout) if ($timeout); -my $h = &make_http_connection($host, $port, $ssl, "GET", $page, \@headers); -alarm(0) if ($timeout); -$h = $main::download_timed_out if ($main::download_timed_out); -if (ref($h)) { - &write_http_connection($h, "\r\n"); - } -&complete_http_download($h, undef, \$error, undef, undef, $host, $port, - undef, $ssl, 1, $timeout); +# Original request my %rs; $rs{'url'} = $url; $rs{'proto'} = $proto; $rs{'host'} = $host; $rs{'port'} = $port; $rs{'page'} = $page; -if (ref($h)) { - $rs{'handle'} = $h - } -else { - $rs{'error'} = $h - } -if (ref($h)) { - - if ($h->{'buffer'} =~ /has\s+moved\s+{'url'} = $rurl; - my ($proto, $host, $uport, $path) = $rurl =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*)/; - my $port = '80' if (!$uport); - if ($proto eq 'https') { - $port = '443' if (!$uport); +eval "use LWP::UserAgent"; +if (!$@) { + my $browser = new LWP::UserAgent(); + $browser->timeout($timeout); + $browser->max_redirect(30); + $browser->agent("Webmin"); + if ($page ne '/') { + $url =~ s/\/$//; + $url .= "$page"; + } + my $response = $browser->get($url); + if (!$response->is_success()) { + $rs{'error'} = $response->status_line; + } + else { + my $request = $response->request(); + if ($request->as_string() =~ /\s((https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*?))\s/mi) { + my $rurl = $1; + my $proto = $2; + my $host = $3; + my $uport = $4; + my $path = $5; + $rurl =~ s/\/$//; + $rs{'redir'}->{'url'} = $rurl; + my $port = '80' if (!$uport); + if ($proto eq 'https') { + $port = '443' if (!$uport); + } + $port ||= $uport; + $rs{'redir'}->{'proto'} = $proto; + $rs{'redir'}->{'host'} = $host; + $rs{'redir'}->{'port'} = $port; + $rs{'redir'}->{'path'} = $path; + } + } +} +else { + # Build headers + my @headers; + push(@headers, [ "Host", $host ]); + push(@headers, [ "User-agent", "Webmin" ]); + push(@headers, [ "Accept-language", "en" ]); + + # Actually download it + $main::download_timed_out = undef; + local $SIG{ALRM} = \&download_timeout; + alarm($timeout); + my $h = &make_http_connection($host, $port, $ssl, "GET", $page, \@headers); + alarm(0); + $h = $main::download_timed_out if ($main::download_timed_out); + if (ref($h)) { + &write_http_connection($h, "\r\n"); + } + &complete_http_download($h, undef, \$error, undef, undef, $host, $port, + undef, $ssl, 1, $timeout); + if (ref($h)) { + $rs{'handle'} = $h + } + else { + $rs{'error'} = $h + } + if (ref($h)) { + if ($h->{'buffer'} =~ /has\s+moved\s+{'url'} = $rurl; + my ($proto, $host, $uport, $path) = $rurl =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*)/; + my $port = '80' if (!$uport); + if ($proto eq 'https') { + $port = '443' if (!$uport); + } + $port ||= $uport; + $rs{'redir'}->{'proto'} = $proto; + $rs{'redir'}->{'host'} = $host; + $rs{'redir'}->{'port'} = $port; + $rs{'redir'}->{'path'} = $path; } - $port ||= $uport; - $rs{'redir'}->{'proto'} = $proto; - $rs{'redir'}->{'host'} = $host; - $rs{'redir'}->{'port'} = $port; - $rs{'redir'}->{'path'} = $path; } } return \%rs; From 96132e28d682324ef5be8f27534a21ea65cfcf8c Mon Sep 17 00:00:00 2001 From: iliajie Date: Wed, 17 May 2023 21:38:36 +0300 Subject: [PATCH 03/15] Fix to test if redirected URL can be resolved --- web-lib-funcs.pl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 571a8ac4c..0c2b8f63d 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13114,6 +13114,20 @@ else { } } } + +# Finally test if redirected URL can be resolved +my $redir_host = $rs{'redir'}->{'host'}; +if ($redir_host) { + my $resolved4 = &to_ipaddress($redir_host); + my $resolved6 = &to_ip6address($redir_host); + if (!$resolved4 && !$resolved6) { + delete $rs{'redir'}; + } + else { + $rs{'redir'}->{'resolved'}->{'ipv4'} = $resolved4; + $rs{'redir'}->{'resolved'}->{'ipv6'} = $resolved6; + } + } return \%rs; } From d308aa8ae834cfa76e73e13d0bbfa74e40b14d1e Mon Sep 17 00:00:00 2001 From: iliajie Date: Wed, 17 May 2023 22:04:43 +0300 Subject: [PATCH 04/15] Fix not to check for SSL certificate --- web-lib-funcs.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 0c2b8f63d..bcb56efa1 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13039,6 +13039,8 @@ eval "use LWP::UserAgent"; if (!$@) { my $browser = new LWP::UserAgent(); $browser->timeout($timeout); + $browser->ssl_opts(verify_hostname => 0, + SSL_verify_mode => 0x00); $browser->max_redirect(30); $browser->agent("Webmin"); if ($page ne '/') { From 46e8076793cdf90c7ee171c301561a8b28d9343a Mon Sep 17 00:00:00 2001 From: iliajie Date: Wed, 17 May 2023 22:23:14 +0300 Subject: [PATCH 05/15] Fix indent --- web-lib-funcs.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index bcb56efa1..41cec1c07 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13072,7 +13072,7 @@ if (!$@) { $rs{'redir'}->{'path'} = $path; } } -} + } else { # Build headers my @headers; From 352e3855dbad339584b2336bf746ae289a78e5aa Mon Sep 17 00:00:00 2001 From: iliajie Date: Wed, 17 May 2023 23:51:38 +0300 Subject: [PATCH 06/15] Fix to only use Webmin code --- web-lib-funcs.pl | 113 ++++++++++++++++------------------------------- 1 file changed, 37 insertions(+), 76 deletions(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 41cec1c07..407755c82 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13035,85 +13035,46 @@ $rs{'proto'} = $proto; $rs{'host'} = $host; $rs{'port'} = $port; $rs{'page'} = $page; -eval "use LWP::UserAgent"; -if (!$@) { - my $browser = new LWP::UserAgent(); - $browser->timeout($timeout); - $browser->ssl_opts(verify_hostname => 0, - SSL_verify_mode => 0x00); - $browser->max_redirect(30); - $browser->agent("Webmin"); - if ($page ne '/') { - $url =~ s/\/$//; - $url .= "$page"; - } - my $response = $browser->get($url); - if (!$response->is_success()) { - $rs{'error'} = $response->status_line; - } - else { - my $request = $response->request(); - if ($request->as_string() =~ /\s((https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*?))\s/mi) { - my $rurl = $1; - my $proto = $2; - my $host = $3; - my $uport = $4; - my $path = $5; - $rurl =~ s/\/$//; - $rs{'redir'}->{'url'} = $rurl; - my $port = '80' if (!$uport); - if ($proto eq 'https') { - $port = '443' if (!$uport); - } - $port ||= $uport; - $rs{'redir'}->{'proto'} = $proto; - $rs{'redir'}->{'host'} = $host; - $rs{'redir'}->{'port'} = $port; - $rs{'redir'}->{'path'} = $path; - } - } + +# Build headers +my @headers; +push(@headers, [ "Host", $host ]); +push(@headers, [ "User-agent", "Webmin" ]); +push(@headers, [ "Accept-language", "en" ]); + +# Actually download it +$main::download_timed_out = undef; +local $SIG{ALRM} = \&download_timeout; +alarm($timeout); +my $h = &make_http_connection($host, $port, $ssl, "GET", $page, \@headers); +alarm(0); +$h = $main::download_timed_out if ($main::download_timed_out); +if (ref($h)) { + &write_http_connection($h, "\r\n"); + } +&complete_http_download($h, undef, \$error, undef, undef, $host, $port, + undef, $ssl, 1, $timeout); +if (ref($h)) { + $rs{'handle'} = $h } else { - # Build headers - my @headers; - push(@headers, [ "Host", $host ]); - push(@headers, [ "User-agent", "Webmin" ]); - push(@headers, [ "Accept-language", "en" ]); - - # Actually download it - $main::download_timed_out = undef; - local $SIG{ALRM} = \&download_timeout; - alarm($timeout); - my $h = &make_http_connection($host, $port, $ssl, "GET", $page, \@headers); - alarm(0); - $h = $main::download_timed_out if ($main::download_timed_out); - if (ref($h)) { - &write_http_connection($h, "\r\n"); - } - &complete_http_download($h, undef, \$error, undef, undef, $host, $port, - undef, $ssl, 1, $timeout); - if (ref($h)) { - $rs{'handle'} = $h - } - else { - $rs{'error'} = $h - } - if (ref($h)) { - if ($h->{'buffer'} =~ /has\s+moved\s+{'url'} = $rurl; - my ($proto, $host, $uport, $path) = $rurl =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*)/; - my $port = '80' if (!$uport); - if ($proto eq 'https') { - $port = '443' if (!$uport); - } - $port ||= $uport; - $rs{'redir'}->{'proto'} = $proto; - $rs{'redir'}->{'host'} = $host; - $rs{'redir'}->{'port'} = $port; - $rs{'redir'}->{'path'} = $path; + $rs{'error'} = $h + } +if (ref($h)) { + if ($h->{'buffer'} =~ /has\s+moved\s+{'url'} = $rurl; + my ($proto, $host, $uport, $path) = $rurl =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*)/; + my $port = '80' if (!$uport); + if ($proto eq 'https') { + $port = '443' if (!$uport); } + $port ||= $uport; + $rs{'redir'}->{'proto'} = $proto; + $rs{'redir'}->{'host'} = $host; + $rs{'redir'}->{'port'} = $port; + $rs{'redir'}->{'path'} = $path; } } From 2436c25a465a56b5098d11e36ccc621801b75614 Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 12:13:45 +0300 Subject: [PATCH 07/15] Fix to reduce timeout --- web-lib-funcs.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 407755c82..247122c1e 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13007,7 +13007,7 @@ The parameters are : =item page - Path in URL, e.g. /about as in https://www.google.com/about -=item timeout - Timeout for connections, defaults to 30s +=item timeout - Timeout for connections, defaults to 15s =cut @@ -13026,7 +13026,7 @@ if ($proto eq 'https') { # Set default port, page and timeout $port ||= $uport; $page ||= '/'; -$timeout ||= 30; +$timeout ||= 15; # Original request my %rs; From 6a693e3dc7172f9a604b8c903ff802d25ec03666 Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 12:14:17 +0300 Subject: [PATCH 08/15] Fix path key name --- web-lib-funcs.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 247122c1e..2a4f7ca57 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13034,7 +13034,7 @@ $rs{'url'} = $url; $rs{'proto'} = $proto; $rs{'host'} = $host; $rs{'port'} = $port; -$rs{'page'} = $page; +$rs{'path'} = $page; # Build headers my @headers; From b1f0c460595b3d9981bae52565d6f66206ca3545 Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 12:14:49 +0300 Subject: [PATCH 09/15] Add support for multi hops in redirects --- web-lib-funcs.pl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 2a4f7ca57..548a6111e 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13014,6 +13014,7 @@ The parameters are : sub get_http_redirect { my ($url, $page, $timeout) = @_; +state %lrs, $ccount++; my ($out, $error, $con_err); my ($proto, $host, $uport) = $url =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?/; my ($ssl, $port) = (0, undef); @@ -13089,8 +13090,21 @@ if ($redir_host) { else { $rs{'redir'}->{'resolved'}->{'ipv4'} = $resolved4; $rs{'redir'}->{'resolved'}->{'ipv6'} = $resolved6; + if ($lrs{'redir'}->{'host'} ne $redir_host) { + $rs{'redir'}->{'hops'} = $ccount; + %lrs = %rs; + my $rport = $rs{'redir'}->{'port'}; + $rport = undef if ($rport =~ /80|443/); + $rport = ":$rport" if ($rport); + my $rpath = $rs{'redir'}->{'path'}; + $rpath = undef if ($rpath eq "/"); + return &get_http_redirect("$proto://$redir_host$rport", $rpath, $timeout); + } } } +if (%lrs) { + %rs = %lrs; + } return \%rs; } From 43edaacada4ae66d26b15c70f35fa614bf2884ed Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 16:43:58 +0300 Subject: [PATCH 10/15] Fix to provide full details on each hop and improve data structures --- web-lib-funcs.pl | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 548a6111e..c298653be 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13014,7 +13014,7 @@ The parameters are : sub get_http_redirect { my ($url, $page, $timeout) = @_; -state %lrs, $ccount++; +state @prs, %lrs, $ccount++; my ($out, $error, $con_err); my ($proto, $host, $uport) = $url =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?/; my ($ssl, $port) = (0, undef); @@ -13036,6 +13036,7 @@ $rs{'proto'} = $proto; $rs{'host'} = $host; $rs{'port'} = $port; $rs{'path'} = $page; +push(@prs, \%rs); # Build headers my @headers; @@ -13056,7 +13057,7 @@ if (ref($h)) { &complete_http_download($h, undef, \$error, undef, undef, $host, $port, undef, $ssl, 1, $timeout); if (ref($h)) { - $rs{'handle'} = $h + $rs{'response'} = $h->{'buffer'}; } else { $rs{'error'} = $h @@ -13088,10 +13089,11 @@ if ($redir_host) { delete $rs{'redir'}; } else { - $rs{'redir'}->{'resolved'}->{'ipv4'} = $resolved4; - $rs{'redir'}->{'resolved'}->{'ipv6'} = $resolved6; + $rs{'redir'}->{'resolved'}->{'ipv4'} = $resolved4 + if ($resolved4); + $rs{'redir'}->{'resolved'}->{'ipv6'} = $resolved6 + if ($resolved6); if ($lrs{'redir'}->{'host'} ne $redir_host) { - $rs{'redir'}->{'hops'} = $ccount; %lrs = %rs; my $rport = $rs{'redir'}->{'port'}; $rport = undef if ($rport =~ /80|443/); @@ -13102,8 +13104,16 @@ if ($redir_host) { } } } -if (%lrs) { - %rs = %lrs; +%rs = %lrs if (keys %lrs); +my %ors = %rs; +%rs = %{$rs{'redir'}}; +if ($ors{'error'}) { + $rs{'error'} = $ors{'error'}; + } +else { + @prs = grep { $_->{'redir'} } @prs; + $rs{'hops'} = \@prs + if (@prs); } return \%rs; } From 377c64ef47db2838ae3b669bf076dc6f9640921e Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 19:18:56 +0300 Subject: [PATCH 11/15] Add comments --- web-lib-funcs.pl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index c298653be..f1af6f4a7 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13009,6 +13009,25 @@ The parameters are : =item timeout - Timeout for connections, defaults to 15s +Example of usage and return data: + + Call: + &get_http_redirect('https://google.com', '/about') + Return: + { + 'hops' => [ + { [...] } + ], + 'host' => 'about.google', + 'path' => '', + 'port' => '443', + 'proto' => 'https', + 'resolved' => { + 'ipv4' => '216.239.32.29' + }, + 'url' => 'https://about.google' + } + =cut sub get_http_redirect From cc15a65c47dcc7310199ef93e3c221d4a6bf4b2f Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 19:50:57 +0300 Subject: [PATCH 12/15] Fir variables declarations --- web-lib-funcs.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index f1af6f4a7..3fc6b7662 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13033,7 +13033,7 @@ Example of usage and return data: sub get_http_redirect { my ($url, $page, $timeout) = @_; -state @prs, %lrs, $ccount++; +state (@prs, %lrs); my ($out, $error, $con_err); my ($proto, $host, $uport) = $url =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?/; my ($ssl, $port) = (0, undef); From 702fc7d5841fe6e0447721fef266b4072fc77491 Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 19:53:24 +0300 Subject: [PATCH 13/15] Fix to return error straight away --- web-lib-funcs.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 3fc6b7662..956ff60c1 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13079,7 +13079,7 @@ if (ref($h)) { $rs{'response'} = $h->{'buffer'}; } else { - $rs{'error'} = $h + return { 'error' => $h }; } if (ref($h)) { if ($h->{'buffer'} =~ /has\s+moved\s+ Date: Thu, 18 May 2023 20:13:24 +0300 Subject: [PATCH 14/15] Add more sense to variables names --- web-lib-funcs.pl | 62 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 956ff60c1..4d83eaa8d 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13033,8 +13033,8 @@ Example of usage and return data: sub get_http_redirect { my ($url, $page, $timeout) = @_; -state (@prs, %lrs); my ($out, $error, $con_err); +state (@hops, %redirects); my ($proto, $host, $uport) = $url =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?/; my ($ssl, $port) = (0, undef); $port = '80' if (!$uport); @@ -13049,13 +13049,13 @@ $page ||= '/'; $timeout ||= 15; # Original request -my %rs; -$rs{'url'} = $url; -$rs{'proto'} = $proto; -$rs{'host'} = $host; -$rs{'port'} = $port; -$rs{'path'} = $page; -push(@prs, \%rs); +my (%redirect); +$redirect{'url'} = $url; +$redirect{'proto'} = $proto; +$redirect{'host'} = $host; +$redirect{'port'} = $port; +$redirect{'path'} = $page; +push(@hops, \%redirect); # Build headers my @headers; @@ -13076,7 +13076,7 @@ if (ref($h)) { &complete_http_download($h, undef, \$error, undef, undef, $host, $port, undef, $ssl, 1, $timeout); if (ref($h)) { - $rs{'response'} = $h->{'buffer'}; + $redirect{'response'} = $h->{'buffer'}; } else { return { 'error' => $h }; @@ -13085,56 +13085,56 @@ if (ref($h)) { if ($h->{'buffer'} =~ /has\s+moved\s+{'url'} = $rurl; + $redirect{'redir'}->{'url'} = $rurl; my ($proto, $host, $uport, $path) = $rurl =~ /^(https?):\/\/([^:\/?#]*)(?:\:([0-9]+))?(.*)/; my $port = '80' if (!$uport); if ($proto eq 'https') { $port = '443' if (!$uport); } $port ||= $uport; - $rs{'redir'}->{'proto'} = $proto; - $rs{'redir'}->{'host'} = $host; - $rs{'redir'}->{'port'} = $port; - $rs{'redir'}->{'path'} = $path; + $redirect{'redir'}->{'proto'} = $proto; + $redirect{'redir'}->{'host'} = $host; + $redirect{'redir'}->{'port'} = $port; + $redirect{'redir'}->{'path'} = $path; } } # Finally test if redirected URL can be resolved -my $redir_host = $rs{'redir'}->{'host'}; +my $redir_host = $redirect{'redir'}->{'host'}; if ($redir_host) { my $resolved4 = &to_ipaddress($redir_host); my $resolved6 = &to_ip6address($redir_host); if (!$resolved4 && !$resolved6) { - delete $rs{'redir'}; + delete $redirect{'redir'}; } else { - $rs{'redir'}->{'resolved'}->{'ipv4'} = $resolved4 + $redirect{'redir'}->{'resolved'}->{'ipv4'} = $resolved4 if ($resolved4); - $rs{'redir'}->{'resolved'}->{'ipv6'} = $resolved6 + $redirect{'redir'}->{'resolved'}->{'ipv6'} = $resolved6 if ($resolved6); - if ($lrs{'redir'}->{'host'} ne $redir_host) { - %lrs = %rs; - my $rport = $rs{'redir'}->{'port'}; + if ($redirects{'redir'}->{'host'} ne $redir_host) { + %redirects = %redirect; + my $rport = $redirect{'redir'}->{'port'}; $rport = undef if ($rport =~ /80|443/); $rport = ":$rport" if ($rport); - my $rpath = $rs{'redir'}->{'path'}; + my $rpath = $redirect{'redir'}->{'path'}; $rpath = undef if ($rpath eq "/"); return &get_http_redirect("$proto://$redir_host$rport", $rpath, $timeout); } } } -%rs = %lrs if (keys %lrs); -my %ors = %rs; -%rs = %{$rs{'redir'}}; -if ($ors{'error'}) { - $rs{'error'} = $ors{'error'}; +%redirect = %redirects if (keys %redirects); +my %original_redirect = %redirect; +%redirect = %{$redirect{'redir'}}; +if ($original_redirect{'error'}) { + $redirect{'error'} = $original_redirect{'error'}; } else { - @prs = grep { $_->{'redir'} } @prs; - $rs{'hops'} = \@prs - if (@prs); + @hops = grep { $_->{'redir'} } @hops; + $redirect{'hops'} = \@hops + if (@hops); } -return \%rs; +return \%redirect; } =head2 get_http_cookie(cookie-name) From 29bb158ef4b5b99f80c7d6f31984792c4d126e1f Mon Sep 17 00:00:00 2001 From: iliajie Date: Thu, 18 May 2023 22:10:28 +0300 Subject: [PATCH 15/15] Fix to drop obsolete code --- web-lib-funcs.pl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/web-lib-funcs.pl b/web-lib-funcs.pl index 4d83eaa8d..ec6a6520a 100755 --- a/web-lib-funcs.pl +++ b/web-lib-funcs.pl @@ -13124,16 +13124,10 @@ if ($redir_host) { } } %redirect = %redirects if (keys %redirects); -my %original_redirect = %redirect; %redirect = %{$redirect{'redir'}}; -if ($original_redirect{'error'}) { - $redirect{'error'} = $original_redirect{'error'}; - } -else { - @hops = grep { $_->{'redir'} } @hops; - $redirect{'hops'} = \@hops - if (@hops); - } +@hops = grep { $_->{'redir'} } @hops; +$redirect{'hops'} = \@hops + if (@hops); return \%redirect; }