From 8e97a8b277b245d738f6e2f90d74d8a8ff7302fb Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 28 Jun 2022 21:33:10 -0400 Subject: [PATCH] Add option to set proxy settings via PAC URL --- examples/raw_parameter_script.py | 1 + seleniumbase/behave/behave_sb.py | 12 +++- seleniumbase/core/browser_launcher.py | 64 +++++++++++++++-- seleniumbase/core/proxy_helper.py | 92 ++++++++++++++++--------- seleniumbase/fixtures/base_case.py | 7 ++ seleniumbase/plugins/pytest_plugin.py | 22 +++++- seleniumbase/plugins/selenium_plugin.py | 18 +++++ 7 files changed, 176 insertions(+), 40 deletions(-) diff --git a/examples/raw_parameter_script.py b/examples/raw_parameter_script.py index 2d056e3d..769e3614 100755 --- a/examples/raw_parameter_script.py +++ b/examples/raw_parameter_script.py @@ -102,6 +102,7 @@ if pure_python: sb.firefox_pref = None sb.proxy_string = None sb.proxy_bypass_list = None + sb.proxy_pac_url = None sb.swiftshader = False sb.ad_block_on = False sb.highlights = None diff --git a/seleniumbase/behave/behave_sb.py b/seleniumbase/behave/behave_sb.py index 21f40960..a2e5ee2d 100644 --- a/seleniumbase/behave/behave_sb.py +++ b/seleniumbase/behave/behave_sb.py @@ -31,6 +31,8 @@ behave -D agent="User Agent String" -D demo -D proxy=SERVER:PORT (Connect to a proxy server:port for tests.) -D proxy=USERNAME:PASSWORD@SERVER:PORT (Use authenticated proxy server.) -D proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com") +-D proxy-pac-url=URL (Connect to a proxy server using a PAC_URL.pac file.) +-D proxy-pac-url=USERNAME:PASSWORD@URL (Authenticated proxy with PAC URL.) -D agent=STRING (Modify the web browser's User-Agent string.) -D mobile (Use the mobile device emulator while running tests.) -D metrics=STRING (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) @@ -195,6 +197,7 @@ def get_configured_sb(context): sb.firefox_pref = None sb.proxy_string = None sb.proxy_bypass_list = None + sb.proxy_pac_url = None sb.swiftshader = False sb.ad_block_on = False sb.highlights = None @@ -619,7 +622,7 @@ def get_configured_sb(context): sb.firefox_pref = firefox_pref continue # Handle: -D proxy=SERVER:PORT / proxy=USERNAME:PASSWORD@SERVER:PORT - if low_key == "proxy": + if low_key in ["proxy", "proxy-server", "proxy-string"]: proxy_string = userdata[key] if proxy_string == "true": proxy_string = sb.proxy_string # revert to default @@ -632,6 +635,13 @@ def get_configured_sb(context): proxy_bypass_list = sb.proxy_bypass_list # revert to default sb.proxy_bypass_list = proxy_bypass_list continue + # Handle: -D proxy-pac-url=URL / proxy-pac-url=USERNAME:PASSWORD@URL + if low_key in ["proxy-pac-url", "proxy_pac_url", "pac-url", "pac_url"]: + proxy_pac_url = userdata[key] + if proxy_pac_url == "true": + proxy_pac_url = sb.proxy_pac_url # revert to default + sb.proxy_pac_url = proxy_pac_url + continue # Handle: -D swiftshader if low_key == "swiftshader": sb.swiftshader = True diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 8a80b2a1..8c8fa73f 100755 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -261,6 +261,7 @@ def _set_chrome_options( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, @@ -458,6 +459,12 @@ def _set_chrome_options( chrome_options.add_argument( "--proxy-bypass-list=%s" % proxy_bypass_list ) + elif proxy_pac_url: + if proxy_auth: + chrome_options = _add_chrome_proxy_extension( + chrome_options, None, proxy_user, proxy_pass + ) + chrome_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url) if headless: if not proxy_auth and not browser_name == constants.Browser.OPERA: # Headless Chrome doesn't support extensions, which are @@ -509,6 +516,7 @@ def _set_firefox_options( locale_code, proxy_string, proxy_bypass_list, + proxy_pac_url, user_agent, disable_csp, firefox_arg, @@ -565,6 +573,9 @@ def _set_firefox_options( options.set_preference("network.proxy.ssl_port", int(proxy_port)) if proxy_bypass_list: options.set_preference("no_proxies_on", proxy_bypass_list) + elif proxy_pac_url: + options.set_preference("network.proxy.type", 2) + options.set_preference("network.proxy.autoconfig_url", proxy_pac_url) if user_agent: options.set_preference("general.useragent.override", user_agent) options.set_preference( @@ -716,6 +727,7 @@ def get_driver( port=4444, proxy_string=None, proxy_bypass_list=None, + proxy_pac_url=None, user_agent=None, cap_file=None, cap_string=None, @@ -775,6 +787,33 @@ def get_driver( proxy_string = validate_proxy_string(proxy_string) if proxy_string and proxy_user and proxy_pass: proxy_auth = True + elif proxy_pac_url: + username_and_password = None + if "@" in proxy_pac_url: + # Format => username:password@PAC_URL.pac + try: + username_and_password = proxy_pac_url.split("@")[0] + proxy_pac_url = proxy_pac_url.split("@")[1] + proxy_user = username_and_password.split(":")[0] + proxy_pass = username_and_password.split(":")[1] + except Exception: + raise Exception( + "The format for using a PAC URL with authentication " + 'is: "username:password@PAC_URL.pac". If using a PAC ' + 'URL without auth, the format is: "PAC_URL.pac".' + ) + if browser_name != constants.Browser.GOOGLE_CHROME and ( + browser_name != constants.Browser.EDGE + ): + raise Exception( + "Chrome or Edge is required when using a PAC URL " + "that has authentication! (If using a PAC URL " + "without auth, Chrome, Edge, or Firefox may be used.)" + ) + if not proxy_pac_url.lower().endswith(".pac"): + raise Exception('The proxy PAC URL must end with ".pac"!') + if proxy_pac_url and proxy_user and proxy_pass: + proxy_auth = True if browser_name == "chrome" and user_data_dir and len(user_data_dir) < 3: raise Exception( "Name length of Chrome's User Data Directory must be >= 3." @@ -792,6 +831,7 @@ def get_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, cap_file, cap_string, @@ -833,6 +873,7 @@ def get_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, @@ -874,6 +915,7 @@ def get_remote_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, cap_file, cap_string, @@ -969,6 +1011,7 @@ def get_remote_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, @@ -1050,6 +1093,7 @@ def get_remote_driver( locale_code, proxy_string, proxy_bypass_list, + proxy_pac_url, user_agent, disable_csp, firefox_arg, @@ -1175,6 +1219,7 @@ def get_remote_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, @@ -1356,6 +1401,7 @@ def get_local_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, @@ -1396,6 +1442,7 @@ def get_local_driver( locale_code, proxy_string, proxy_bypass_list, + proxy_pac_url, user_agent, disable_csp, firefox_arg, @@ -1704,10 +1751,16 @@ def get_local_driver( edge_options, proxy_string, proxy_user, proxy_pass ) edge_options.add_argument("--proxy-server=%s" % proxy_string) - if proxy_bypass_list: - edge_options.add_argument( - "--proxy-bypass-list=%s" % proxy_bypass_list - ) + if proxy_bypass_list: + edge_options.add_argument( + "--proxy-bypass-list=%s" % proxy_bypass_list + ) + elif proxy_pac_url: + if proxy_auth: + edge_options = _add_chrome_proxy_extension( + edge_options, None, proxy_user, proxy_pass + ) + edge_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url) edge_options.add_argument("--test-type") edge_options.add_argument("--log-level=3") edge_options.add_argument("--no-first-run") @@ -1862,6 +1915,7 @@ def get_local_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, @@ -1917,6 +1971,7 @@ def get_local_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, @@ -2036,6 +2091,7 @@ def get_local_driver( proxy_user, proxy_pass, proxy_bypass_list, + proxy_pac_url, user_agent, recorder_ext, disable_csp, diff --git a/seleniumbase/core/proxy_helper.py b/seleniumbase/core/proxy_helper.py index 869f5c99..38bd359f 100755 --- a/seleniumbase/core/proxy_helper.py +++ b/seleniumbase/core/proxy_helper.py @@ -11,39 +11,65 @@ def create_proxy_zip(proxy_string, proxy_user, proxy_pass): """Implementation of https://stackoverflow.com/a/35293284 for https://stackoverflow.com/questions/12848327/ (Run Selenium on a proxy server that requires authentication.) - Solution involves creating & adding a Chrome extension on the fly. - * CHROME-ONLY for now! * + Solution involves creating & adding a Chromium extension on the fly. + CHROMIUM-ONLY! *** Only Chrome and Edge browsers are supported. *** """ - proxy_host = proxy_string.split(":")[0] - proxy_port = proxy_string.split(":")[1] - background_js = ( - """var config = {\n""" - """ mode: "fixed_servers",\n""" - """ rules: {\n""" - """ singleProxy: {\n""" - """ scheme: "http",\n""" - """ host: "%s",\n""" - """ port: parseInt("%s")\n""" - """ },\n""" - """ }\n""" - """ };\n""" - """chrome.proxy.settings.set(""" - """{value: config, scope: "regular"}, function() {""" - """});\n""" - """function callbackFn(details) {\n""" - """ return {\n""" - """ authCredentials: {\n""" - """ username: "%s",\n""" - """ password: "%s"\n""" - """ }\n""" - """ };\n""" - """}\n""" - """chrome.webRequest.onAuthRequired.addListener(\n""" - """ callbackFn,\n""" - """ {urls: [""]},\n""" - """ ['blocking']\n""" - """);""" % (proxy_host, proxy_port, proxy_user, proxy_pass) - ) + background_js = None + if proxy_string: + proxy_host = proxy_string.split(":")[0] + proxy_port = proxy_string.split(":")[1] + background_js = ( + """var config = {\n""" + """ mode: "fixed_servers",\n""" + """ rules: {\n""" + """ singleProxy: {\n""" + """ scheme: "http",\n""" + """ host: "%s",\n""" + """ port: parseInt("%s")\n""" + """ },\n""" + """ }\n""" + """ };\n""" + """chrome.proxy.settings.set(""" + """{value: config, scope: "regular"}, function() {""" + """});\n""" + """function callbackFn(details) {\n""" + """ return {\n""" + """ authCredentials: {\n""" + """ username: "%s",\n""" + """ password: "%s"\n""" + """ }\n""" + """ };\n""" + """}\n""" + """chrome.webRequest.onAuthRequired.addListener(\n""" + """ callbackFn,\n""" + """ {urls: [""]},\n""" + """ ['blocking']\n""" + """);""" % (proxy_host, proxy_port, proxy_user, proxy_pass) + ) + else: + background_js = ( + """var config = {\n""" + """ mode: "fixed_servers",\n""" + """ rules: {\n""" + """ }\n""" + """ };\n""" + """chrome.proxy.settings.set(""" + """{value: config, scope: "regular"}, function() {""" + """});\n""" + """function callbackFn(details) {\n""" + """ return {\n""" + """ authCredentials: {\n""" + """ username: "%s",\n""" + """ password: "%s"\n""" + """ }\n""" + """ };\n""" + """}\n""" + """chrome.webRequest.onAuthRequired.addListener(\n""" + """ callbackFn,\n""" + """ {urls: [""]},\n""" + """ ['blocking']\n""" + """);""" % (proxy_user, proxy_pass) + ) manifest_json = ( """{\n""" """"version": "1.0.0",\n""" @@ -79,7 +105,7 @@ def create_proxy_zip(proxy_string, proxy_user, proxy_pass): def remove_proxy_zip_if_present(): - """Remove Chrome extension zip file used for proxy server authentication. + """Remove Chromium extension zip file used for proxy server authentication. Used in the implementation of https://stackoverflow.com/a/35293284 for https://stackoverflow.com/questions/12848327/ """ diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 02406c1d..8f3e9c9a 100755 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -2937,6 +2937,7 @@ class BaseCase(unittest.TestCase): port=None, proxy=None, proxy_bypass_list=None, + proxy_pac_url=None, agent=None, switch_to=True, cap_file=None, @@ -2980,6 +2981,7 @@ class BaseCase(unittest.TestCase): port - if using a Selenium Grid, set the host port here proxy - if using a proxy server, specify the "host:port" combo here proxy_bypass_list - ";"-separated hosts to bypass (Eg. "*.foo.com") + proxy_pac_url - designates the proxy PAC URL to use (Chromium-only) switch_to - the option to switch to the new driver (default = True) cap_file - the file containing desired capabilities for the browser cap_string - the string with desired capabilities for the browser @@ -3060,6 +3062,8 @@ class BaseCase(unittest.TestCase): proxy_string = self.proxy_string if proxy_bypass_list is None: proxy_bypass_list = self.proxy_bypass_list + if proxy_pac_url is None: + proxy_pac_url = self.proxy_pac_url user_agent = agent if user_agent is None: user_agent = self.user_agent @@ -3137,6 +3141,7 @@ class BaseCase(unittest.TestCase): port=port, proxy_string=proxy_string, proxy_bypass_list=proxy_bypass_list, + proxy_pac_url=proxy_pac_url, user_agent=user_agent, cap_file=cap_file, cap_string=cap_string, @@ -12060,6 +12065,7 @@ class BaseCase(unittest.TestCase): self.port = sb_config.port self.proxy_string = sb_config.proxy_string self.proxy_bypass_list = sb_config.proxy_bypass_list + self.proxy_pac_url = sb_config.proxy_pac_url self.user_agent = sb_config.user_agent self.mobile_emulator = sb_config.mobile_emulator self.device_metrics = sb_config.device_metrics @@ -12372,6 +12378,7 @@ class BaseCase(unittest.TestCase): port=self.port, proxy=self.proxy_string, proxy_bypass_list=self.proxy_bypass_list, + proxy_pac_url=self.proxy_pac_url, agent=self.user_agent, switch_to=True, cap_file=self.cap_file, diff --git a/seleniumbase/plugins/pytest_plugin.py b/seleniumbase/plugins/pytest_plugin.py index d149803a..35a04823 100644 --- a/seleniumbase/plugins/pytest_plugin.py +++ b/seleniumbase/plugins/pytest_plugin.py @@ -43,6 +43,8 @@ def pytest_addoption(parser): --proxy=SERVER:PORT (Connect to a proxy server:port for tests.) --proxy=USERNAME:PASSWORD@SERVER:PORT (Use authenticated proxy server.) --proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com") + --proxy-pac-url=URL (Connect to a proxy server using a PAC_URL.pac file.) + --proxy-pac-url=USERNAME:PASSWORD@URL (Authenticated proxy with PAC URL.) --agent=STRING (Modify the web browser's User-Agent string.) --mobile (Use the mobile device emulator while running tests.) --metrics=STRING (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) @@ -410,13 +412,15 @@ def pytest_addoption(parser): ) parser.addoption( "--proxy", + "--proxy-server", + "--proxy-string", action="store", dest="proxy_string", default=None, help="""Designates the proxy server:port to use. Format: servername:port. OR - username:password@servername:port OR - A dict key from proxy_list.PROXY_LIST + username:password@servername:port OR + A dict key from proxy_list.PROXY_LIST Default: None.""", ) parser.addoption( @@ -437,6 +441,19 @@ def pytest_addoption(parser): --proxy-bypass-list="127.0.0.1:8080" Default: None.""", ) + parser.addoption( + "--proxy-pac-url", + "--proxy_pac_url", + "--pac-url", + "--pac_url", + action="store", + dest="proxy_pac_url", + default=None, + help="""Designates the proxy PAC URL to use. + Format: A URL string OR + A username:password@URL string + Default: None.""", + ) parser.addoption( "--agent", "--user-agent", @@ -1176,6 +1193,7 @@ def pytest_configure(config): sb_config.protocol = "https" sb_config.proxy_string = config.getoption("proxy_string") sb_config.proxy_bypass_list = config.getoption("proxy_bypass_list") + sb_config.proxy_pac_url = config.getoption("proxy_pac_url") sb_config.cap_file = config.getoption("cap_file") sb_config.cap_string = config.getoption("cap_string") sb_config.settings_file = config.getoption("settings_file") diff --git a/seleniumbase/plugins/selenium_plugin.py b/seleniumbase/plugins/selenium_plugin.py index 12809ea9..624e2be2 100755 --- a/seleniumbase/plugins/selenium_plugin.py +++ b/seleniumbase/plugins/selenium_plugin.py @@ -26,6 +26,8 @@ class SeleniumBrowser(Plugin): --proxy=SERVER:PORT (Connect to a proxy server:port for tests.) --proxy=USERNAME:PASSWORD@SERVER:PORT (Use authenticated proxy server.) --proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com") + --proxy-pac-url=URL (Connect to a proxy server using a PAC_URL.pac file.) + --proxy-pac-url=USERNAME:PASSWORD@URL (Authenticated proxy with PAC URL.) --agent=STRING (Modify the web browser's User-Agent string.) --mobile (Use the mobile device emulator while running tests.) --metrics=STRING (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) @@ -160,6 +162,8 @@ class SeleniumBrowser(Plugin): ) parser.add_option( "--proxy", + "--proxy-server", + "--proxy-string", action="store", dest="proxy_string", default=None, @@ -187,6 +191,19 @@ class SeleniumBrowser(Plugin): --proxy-bypass-list="127.0.0.1:8080" Default: None.""", ) + parser.add_option( + "--proxy-pac-url", + "--proxy_pac_url", + "--pac-url", + "--pac_url", + action="store", + dest="proxy_pac_url", + default=None, + help="""Designates the proxy PAC URL to use. + Format: A URL string OR + A username:password@URL string + Default: None.""", + ) parser.add_option( "--agent", "--user-agent", @@ -743,6 +760,7 @@ class SeleniumBrowser(Plugin): test.test.firefox_pref = self.options.firefox_pref test.test.proxy_string = self.options.proxy_string test.test.proxy_bypass_list = self.options.proxy_bypass_list + test.test.proxy_pac_url = self.options.proxy_pac_url test.test.user_agent = self.options.user_agent test.test.mobile_emulator = self.options.mobile_emulator test.test.device_metrics = self.options.device_metrics