From a44a1d5395d9ad3538c02e6da66da9cb53a298b2 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Fri, 24 Jan 2025 17:38:38 -0500 Subject: [PATCH] Add support for downloading browsers --- .gitignore | 14 ++ help_docs/webdriver_installation.md | 34 ++- seleniumbase/console_scripts/ReadMe.md | 6 +- seleniumbase/console_scripts/run.py | 8 +- seleniumbase/console_scripts/sb_install.py | 241 ++++++++++++++++++++- seleniumbase/drivers/ReadMe.md | 48 +++- seleniumbase/fixtures/constants.py | 3 + 7 files changed, 325 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 86fc05b1..5d854999 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,20 @@ msedgedriver.exe operadriver.exe uc_driver.exe +# Chrome for Testing folders +chrome-mac-arm64 +chrome-mac-x64 +chrome-linux64 +chrome-win64 +chrome-win32 + +# Chrome-Headless-Shell folders +chrome-headless-shell-mac-arm64 +chrome-headless-shell-mac-x64 +chrome-headless-shell-linux64 +chrome-headless-shell-win64 +chrome-headless-shell-win32 + # msedgedriver requirements libc++.dylib diff --git a/help_docs/webdriver_installation.md b/help_docs/webdriver_installation.md index f5b8dd16..cbae48fa 100644 --- a/help_docs/webdriver_installation.md +++ b/help_docs/webdriver_installation.md @@ -4,7 +4,7 @@ To run web automation, you need webdrivers for each browser you plan on using. With SeleniumBase, drivers are downloaded automatically (as needed) into the SeleniumBase `drivers/` folder. -You can also download drivers manually with these commands: +🎛️ You can also download drivers manually with these commands: ```bash seleniumbase get chromedriver @@ -16,7 +16,7 @@ After running the commands above, web drivers will get downloaded into the `sele If the necessary driver is not found in this location while running tests, SeleniumBase will instead look for the driver on the System PATH. If the necessary driver is not on the System PATH either, SeleniumBase will automatically attempt to download the required driver. -* You can also download specific versions of drivers. Examples: +🎛️ You can also download specific versions of drivers. Examples: ```bash sbase get chromedriver 114 @@ -30,7 +30,7 @@ sbase get chromedriver mlatest # Milestone latest version for detected browser sbase get edgedriver 115.0.1901.183 ``` -(NOTE: ``sbase`` is a shortcut for ``seleniumbase``) +(NOTE: `sbase` is a shortcut for `seleniumbase`) -------- @@ -48,7 +48,7 @@ Here's where you can go to manually get web drivers from the source: **macOS shortcuts**: -* You can also install drivers by using ``brew`` (aka ``homebrew``): +🎛️ You can also install drivers by using ``brew`` (aka ``homebrew``): ```bash brew install --cask chromedriver @@ -56,7 +56,7 @@ brew install --cask chromedriver brew install geckodriver ``` -You can also upgrade existing webdrivers: +🎛️ You can also upgrade existing webdrivers: ```bash brew upgrade --cask chromedriver @@ -66,7 +66,7 @@ brew upgrade geckodriver **Linux shortcuts**: -If you still need drivers, these scripts download ``chromedriver`` and ``geckodriver`` to a Linux machine: +🎛️ If you still need drivers, these scripts download `chromedriver` and `geckodriver` to a Linux machine: ```bash wget https://chromedriver.storage.googleapis.com/114.0.5735.90/chromedriver_linux64.zip @@ -76,12 +76,30 @@ chmod +x /usr/local/bin/chromedriver ``` ```bash -wget https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz -tar xvfz geckodriver-v0.34.0-linux64.tar.gz +wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz +tar xvfz geckodriver-v0.35.0-linux64.tar.gz mv geckodriver /usr/local/bin/ chmod +x /usr/local/bin/geckodriver ``` To verify that web drivers are working, **[follow these instructions](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/verify_webdriver.md)**. +-------- + +**Browser Binaries**: + +🎛️ Use the `sbase get` command to download the `Chrome for Testing` and `Chrome-Headless-Shell` browser binaries. Example: + +```bash +sbase get cft # (For `Chrome for Testing`) +sbase get chs # (For `Chrome-Headless-Shell`) +``` + +Those commands download those binaries into the `seleniumbase/drivers` folder. +To use the binaries from there in SeleniumBase scripts, set the `binary_location` to `cft` or `chs`. + +(Source: https://googlechromelabs.github.io/chrome-for-testing/) + +-------- + [](https://github.com/seleniumbase/SeleniumBase) diff --git a/seleniumbase/console_scripts/ReadMe.md b/seleniumbase/console_scripts/ReadMe.md index 43faa057..58cf7190 100644 --- a/seleniumbase/console_scripts/ReadMe.md +++ b/seleniumbase/console_scripts/ReadMe.md @@ -68,10 +68,12 @@ sbase get chromedriver 114.0.5735.90 sbase get chromedriver stable sbase get chromedriver beta sbase get chromedriver -p +sbase get cft 131 +sbase get chs ``` -(Drivers: ``chromedriver``, ``geckodriver``, ``edgedriver``, - ``iedriver``, ``uc_driver``) +(Drivers: ``chromedriver``, ``cft``, ``uc_driver``, + ``edgedriver``, ``chs``, ``geckodriver``) (Options: A specific driver version or major version integer. If not set, the driver version matches the browser. diff --git a/seleniumbase/console_scripts/run.py b/seleniumbase/console_scripts/run.py index 57c15cb0..962a7ad5 100644 --- a/seleniumbase/console_scripts/run.py +++ b/seleniumbase/console_scripts/run.py @@ -139,8 +139,8 @@ def show_install_usage(): print(" OR: seleniumbase get [DRIVER_NAME] [OPTIONS]") print(" OR: sbase install [DRIVER_NAME] [OPTIONS]") print(" OR: sbase get [DRIVER_NAME] [OPTIONS]") - print(" (Drivers: chromedriver, geckodriver,") - print(" edgedriver, iedriver, uc_driver)") + print(" (Drivers: chromedriver, cft, uc_driver,") + print(" edgedriver, chs, geckodriver)") print(" Options:") print(" VERSION Specify the version to download.") print(" Tries to detect the needed version.") @@ -157,11 +157,15 @@ def show_install_usage(): print(" sbase get chromedriver stable") print(" sbase get chromedriver beta") print(" sbase get chromedriver -p") + print(" sbase get cft 131") + print(" sbase get chs") print(" Output:") print(" Downloads the webdriver to seleniumbase/drivers/") print(" (chromedriver is required for Chrome automation)") print(" (geckodriver is required for Firefox automation)") print(" (edgedriver is required for MS__Edge automation)") + print(" (cft is for the `Chrome for Testing` binary exe)") + print(" (chs is for the `Chrome-Headless-Shell` binary.)") print("") diff --git a/seleniumbase/console_scripts/sb_install.py b/seleniumbase/console_scripts/sb_install.py index 8d55ac87..5d6de44b 100644 --- a/seleniumbase/console_scripts/sb_install.py +++ b/seleniumbase/console_scripts/sb_install.py @@ -3,7 +3,7 @@ Downloads the specified webdriver to "seleniumbase/drivers/" Usage: sbase get {chromedriver|geckodriver|edgedriver| - iedriver|uc_driver} [OPTIONS] + iedriver|uc_driver|cft|chs} [OPTIONS] Options: VERSION Specify the version. Tries to detect the needed version. @@ -19,6 +19,8 @@ Examples: sbase get chromedriver stable sbase get chromedriver beta sbase get chromedriver -p + sbase get cft 131 + sbase get chs Output: Downloads the webdriver to seleniumbase/drivers/ (chromedriver is required for Chrome automation) @@ -31,6 +33,7 @@ import os import platform import requests import shutil +import subprocess import sys import time import tarfile @@ -61,8 +64,8 @@ def invalid_run_command(): exp += " OR sbase install [DRIVER_NAME] [OPTIONS]\n" exp += " OR seleniumbase get [DRIVER_NAME] [OPTIONS]\n" exp += " OR sbase get [DRIVER_NAME] [OPTIONS]\n" - exp += " (Drivers: chromedriver, geckodriver,\n" - exp += " edgedriver, iedriver, uc_driver)\n" + exp += " (Drivers: chromedriver, cft, uc_driver,\n" + exp += " edgedriver, chs, geckodriver)\n" exp += " Options:\n" exp += " VERSION Specify the version.\n" exp += " Tries to detect the needed version.\n" @@ -79,11 +82,15 @@ def invalid_run_command(): exp += " sbase get chromedriver stable\n" exp += " sbase get chromedriver beta\n" exp += " sbase get chromedriver -p\n" + exp += " sbase get cft 131\n" + exp += " sbase get chs\n" exp += " Output:\n" exp += " Downloads the webdriver to seleniumbase/drivers/\n" exp += " (chromedriver is required for Chrome automation)\n" exp += " (geckodriver is required for Firefox automation)\n" exp += " (edgedriver is required for MS__Edge automation)\n" + exp += " (cft is for the `Chrome for Testing` binary exe)\n" + exp += " (chs is for the `Chrome-Headless-Shell` binary.)\n" print("") raise Exception("%s\n\n%s" % (constants.Warnings.INVALID_RUN_COMMAND, exp)) @@ -265,6 +272,16 @@ def main(override=None, intel_for_uc=None, force_uc=None): elif override.startswith("iedriver "): extra = override.split("iedriver ")[1] sys.argv = ["seleniumbase", "get", "iedriver", extra] + elif override == "cft": + sys.argv = ["seleniumbase", "get", "cft"] + elif override.startswith("cft "): + extra = override.split("cft ")[1] + sys.argv = ["seleniumbase", "get", "cft", extra] + elif override == "chs": + sys.argv = ["seleniumbase", "get", "chs"] + elif override.startswith("chs "): + extra = override.split("chs ")[1] + sys.argv = ["seleniumbase", "get", "chs", extra] if found_proxy: sys.argv.append(found_proxy) @@ -550,6 +567,134 @@ def main(override=None, intel_for_uc=None, force_uc=None): raise Exception("Could not find chromedriver to download!\n") if not get_latest: pass + elif name == "chrome" or name == "cft": + set_version = None + found_version = None + use_version = None + major_version = None + if num_args >= 4: + set_version = sys.argv[3] + if ( + set_version + and set_version.split(".")[0].isnumeric() + and int(set_version.split(".")[0]) >= 113 + ): + major_version = set_version.split(".")[0] + elif ( + not set_version + or set_version.lower() == "latest" + or set_version.lower() == "stable" + ): + found_version = get_latest_stable_chromedriver_version() + elif ( + set_version and ( + set_version.lower() == "latest-1" + or set_version.lower() == "previous" + ) + ): + found_version = get_latest_stable_chromedriver_version() + major_version = str(int(found_version.split(".")[0]) - 1) + found_version = None + elif (set_version and set_version.lower() == "beta"): + found_version = get_latest_beta_chromedriver_version() + elif (set_version and set_version.lower() == "dev"): + found_version = get_latest_dev_chromedriver_version() + elif (set_version and set_version.lower() == "canary"): + found_version = get_latest_canary_chromedriver_version() + if found_version and found_version.split(".")[0].isnumeric(): + major_version = found_version.split(".")[0] + use_version = found_version + if not use_version: + use_version = get_cft_latest_version_from_milestone(major_version) + msg = c2 + "Chrome for Testing to download" + cr + p_version = c3 + use_version + cr + log_d("\n*** %s = %s" % (msg, p_version)) + if IS_MAC: + if IS_ARM_MAC: + platform_code = "mac-arm64" + file_name = "chrome-mac-arm64.zip" + else: + platform_code = "mac-x64" + file_name = "chrome-mac-x64.zip" + elif IS_LINUX: + platform_code = "linux64" + file_name = "chrome-linux64.zip" + elif IS_WINDOWS: + if "64" in ARCH: + platform_code = "win64" + file_name = "chrome-win64.zip" + else: + platform_code = "win32" + file_name = "chrome-win32.zip" + plat_arch = file_name.split(".zip")[0] + download_url = ( + "https://storage.googleapis.com/chrome-for-testing-public/" + "%s/%s/%s" % (use_version, platform_code, file_name) + ) + elif name == "chrome-headless-shell" or name == "chs": + set_version = None + found_version = None + use_version = None + major_version = None + if num_args >= 4: + set_version = sys.argv[3] + if ( + set_version + and set_version.split(".")[0].isnumeric() + and int(set_version.split(".")[0]) >= 113 + ): + major_version = set_version.split(".")[0] + elif ( + not set_version + or set_version.lower() == "latest" + or set_version.lower() == "stable" + ): + found_version = get_latest_stable_chromedriver_version() + elif ( + set_version and ( + set_version.lower() == "latest-1" + or set_version.lower() == "previous" + ) + ): + found_version = get_latest_stable_chromedriver_version() + major_version = str(int(found_version.split(".")[0]) - 1) + found_version = None + elif (set_version and set_version.lower() == "beta"): + found_version = get_latest_beta_chromedriver_version() + elif (set_version and set_version.lower() == "dev"): + found_version = get_latest_dev_chromedriver_version() + elif (set_version and set_version.lower() == "canary"): + found_version = get_latest_canary_chromedriver_version() + if found_version and found_version.split(".")[0].isnumeric(): + major_version = found_version.split(".")[0] + use_version = found_version + if not use_version: + use_version = get_cft_latest_version_from_milestone(major_version) + msg = c2 + "Chrome-Headless-Shell to download" + cr + p_version = c3 + use_version + cr + log_d("\n*** %s = %s" % (msg, p_version)) + if IS_MAC: + if IS_ARM_MAC: + platform_code = "mac-arm64" + file_name = "chrome-headless-shell-mac-arm64.zip" + else: + platform_code = "mac-x64" + file_name = "chrome-headless-shell-mac-x64.zip" + elif IS_LINUX: + platform_code = "linux64" + file_name = "chrome-headless-shell-linux64.zip" + elif IS_WINDOWS: + if "64" in ARCH: + platform_code = "win64" + file_name = "chrome-headless-shell-win64.zip" + else: + platform_code = "win32" + file_name = "chrome-headless-shell-win32.zip" + plat_arch = file_name.split(".zip")[0] + download_url = ( + "https://storage.googleapis.com/chrome-for-testing-public/" + "%s/%s/%s" % (use_version, platform_code, file_name) + ) elif name == "geckodriver" or name == "firefoxdriver": use_version = DEFAULT_GECKODRIVER_VERSION found_geckodriver = False @@ -1029,10 +1174,14 @@ def main(override=None, intel_for_uc=None, force_uc=None): if os.path.exists(os.path.join(downloads_folder, "Driver_Notes/")): # Only works if the directory is empty os.rmdir(os.path.join(downloads_folder, "Driver_Notes/")) - pr_driver_path = c3 + driver_path + cr + driver_base = os.sep.join(driver_path.split(os.sep)[:-1]) + driver_file = driver_path.split(os.sep)[-1] + pr_driver_base = c3 + driver_base + cr + pr_sep = c3 + os.sep + cr + pr_driver_file = c3 + driver_file + cr log_d( - "The file [%s] was saved to:\n%s\n" - % (driver_file, pr_driver_path) + "The file [%s] was saved to:\n%s%s\n%s\n" + % (driver_file, pr_driver_base, pr_sep, pr_driver_file) ) log_d("Making [%s %s] executable ..." % (driver_file, use_version)) make_executable(driver_path) @@ -1046,6 +1195,86 @@ def main(override=None, intel_for_uc=None, force_uc=None): make_executable(path_file) log_d("Also copied to: %s%s%s" % (c3, path_file, cr)) log_d("") + elif name == "chrome" or name == "cft": + # Zip file is valid. Proceed. + driver_path = None + driver_file = None + base_path = os.sep.join(zip_file_path.split(os.sep)[:-1]) + folder_name = contents[0].split(os.sep)[0] + folder_path = os.path.join(base_path, folder_name) + if IS_MAC or IS_LINUX: + if ( + "chrome-" in folder_path + and "drivers" in folder_path + and os.path.exists(folder_path) + ): + shutil.rmtree(folder_path) + subprocess.run( + ["unzip", zip_file_path, "-d", downloads_folder] + ) + elif IS_WINDOWS: + subprocess.run( + [ + "powershell", + "Expand-Archive", + "-Path", + zip_file_path, + "-DestinationPath", + downloads_folder, + ] + ) + else: + zip_ref.extractall(downloads_folder) + zip_ref.close() + os.remove(zip_file_path) + log_d("%sUnzip Complete!%s\n" % (c2, cr)) + pr_base_path = c3 + base_path + cr + pr_sep = c3 + os.sep + cr + pr_folder_name = c3 + folder_name + cr + log_d( + "Chrome for Testing was saved inside:\n%s%s\n%s\n" + % (pr_base_path, pr_sep, pr_folder_name) + ) + elif name == "chrome-headless-shell" or name == "chs": + # Zip file is valid. Proceed. + driver_path = None + driver_file = None + base_path = os.sep.join(zip_file_path.split(os.sep)[:-1]) + folder_name = contents[0].split(os.sep)[0] + folder_path = os.path.join(base_path, folder_name) + if IS_MAC or IS_LINUX: + if ( + "chrome-headless-shell-" in folder_path + and "drivers" in folder_path + and os.path.exists(folder_path) + ): + shutil.rmtree(folder_path) + subprocess.run( + ["unzip", zip_file_path, "-d", downloads_folder] + ) + elif IS_WINDOWS: + subprocess.run( + [ + "powershell", + "Expand-Archive", + "-Path", + zip_file_path, + "-DestinationPath", + downloads_folder, + ] + ) + else: + zip_ref.extractall(downloads_folder) + zip_ref.close() + os.remove(zip_file_path) + log_d("%sUnzip Complete!%s\n" % (c2, cr)) + pr_base_path = c3 + base_path + cr + pr_sep = c3 + os.sep + cr + pr_folder_name = c3 + folder_name + cr + log_d( + "Chrome-Headless-Shell was saved inside:\n%s%s\n%s\n" + % (pr_base_path, pr_sep, pr_folder_name) + ) elif len(contents) == 0: raise Exception("Zip file %s is empty!" % zip_file_path) else: diff --git a/seleniumbase/drivers/ReadMe.md b/seleniumbase/drivers/ReadMe.md index 0ac22a01..bb699e57 100644 --- a/seleniumbase/drivers/ReadMe.md +++ b/seleniumbase/drivers/ReadMe.md @@ -1,8 +1,10 @@ -### SeleniumBase webdriver storage + -To run web automation, you'll need webdrivers for each browser you plan on using. With SeleniumBase, drivers are downloaded automatically as needed into the SeleniumBase ``drivers`` folder. +## SeleniumBase driver storage -You can also download drivers manually with these commands: +To run web automation, you'll need webdrivers for each browser you plan on using. With SeleniumBase, drivers are downloaded automatically as needed into the SeleniumBase `drivers` folder. + +🎛️ You can also download drivers manually with these commands: ```bash seleniumbase get chromedriver @@ -10,18 +12,42 @@ seleniumbase get geckodriver seleniumbase get edgedriver ``` -After running the commands above, web drivers will get downloaded into the ``seleniumbase/drivers/`` folder. SeleniumBase uses those drivers during tests. (The drivers don't come with SeleniumBase by default.) +After running the commands above, web drivers will get downloaded into the `seleniumbase/drivers/` folder. SeleniumBase uses those drivers during tests. (The drivers don't come with SeleniumBase by default.) If the necessary driver is not found in this location while running tests, SeleniumBase will instead look for the driver on the System PATH. If the necessary driver is not on the System PATH either, SeleniumBase will automatically attempt to download the required driver. -* You can also download specific versions of drivers. Examples: +🎛️ You can also download specific versions of drivers. Examples: ```bash -sbase get chromedriver 107 -sbase get chromedriver 107.0.5304.62 -sbase get chromedriver latest -sbase get chromedriver latest-1 -sbase get edgedriver 106.0.1370.42 +sbase get chromedriver 114 +sbase get chromedriver 114.0.5735.90 +sbase get chromedriver stable +sbase get chromedriver beta +sbase get chromedriver dev +sbase get chromedriver canary +sbase get chromedriver previous # One major version before the stable version +sbase get chromedriver mlatest # Milestone latest version for detected browser +sbase get edgedriver 115.0.1901.183 ``` -(NOTE: ``sbase`` is a shortcut for ``seleniumbase``) +(NOTE: `sbase` is a shortcut for `seleniumbase`) + +-------- + +**Browser Binaries**: + +🎛️ Use the `sbase get` command to download the `Chrome for Testing` and `Chrome-Headless-Shell` browser binaries. Example: + +```bash +sbase get cft # (For `Chrome for Testing`) +sbase get chs # (For `Chrome-Headless-Shell`) +``` + +Those commands download those binaries into the `seleniumbase/drivers` folder. +To use the binaries from there in SeleniumBase scripts, set the `binary_location` to `cft` or `chs`. + +(Source: https://googlechromelabs.github.io/chrome-for-testing/) + +-------- + +[](https://github.com/seleniumbase/SeleniumBase) diff --git a/seleniumbase/fixtures/constants.py b/seleniumbase/fixtures/constants.py index 9fd3131c..16d300a4 100644 --- a/seleniumbase/fixtures/constants.py +++ b/seleniumbase/fixtures/constants.py @@ -400,6 +400,7 @@ class ValidBinaries: "google-chrome-beta", "google-chrome-dev", "google-chrome-unstable", + "chrome-headless-shell", "brave-browser", "brave-browser-stable", "brave", @@ -418,6 +419,7 @@ class ValidBinaries: "Google Chrome", "Chromium", "Google Chrome for Testing", + "chrome-headless-shell", "Google Chrome Beta", "Google Chrome Dev", "Brave Browser", @@ -429,6 +431,7 @@ class ValidBinaries: valid_chrome_binaries_on_windows = [ "chrome.exe", "chromium.exe", + "chrome-headless-shell.exe", "brave.exe", "opera.exe", ]