From f003b51be9da6aa7c1f8e7fcc4a213ea2ffe56c4 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Sat, 14 Dec 2024 12:18:20 -0600 Subject: [PATCH] feat(python): add other indygreg flavors (#3565) Fixes https://github.com/jdx/mise/issues/2149 --- Cargo.lock | 1 + Cargo.toml | 1 + docs/lang/python.md | 8 ++++++-- schema/mise.json | 4 ++++ settings.toml | 10 ++++++++++ src/file.rs | 26 ++++++++++++++++++++------ src/plugins/core/python.rs | 35 ++++++++++++++++++++++++----------- tasks.md | 4 ++++ 8 files changed, 70 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 152c1db51..83393b349 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2225,6 +2225,7 @@ dependencies = [ "xx", "xz2", "zip", + "zstd", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cd1f3f27a..edfe71501 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ which = "7" xx = { version = "2", features = ["glob"] } xz2 = "0.1" zip = { version = "2", default-features = false, features = ["deflate"] } +zstd = "0.13" [target.'cfg(unix)'.dependencies] exec = "0.3" diff --git a/docs/lang/python.md b/docs/lang/python.md index d58544e71..782fe68e4 100644 --- a/docs/lang/python.md +++ b/docs/lang/python.md @@ -148,7 +148,11 @@ The venv will need to be created manually with `python -m venv /path/to/venv` un Free-threaded python can be installed via python-build by running the following: ```bash -MISE_PYTHON_COMPILE=1 PYTHON_BUILD_FREE_THREADING=1 mise install +MISE_PYTHON_COMPILE=0 MISE_PYTHON_PRECOMPILED_FLAVOR=freethreaded+pgo-full mise install python ``` -Currently, they are not supported with precompiled binaries. +Or to compile with python-build: + +```bash +MISE_PYTHON_COMPILE=1 PYTHON_BUILD_FREE_THREADING=1 mise install python +``` diff --git a/schema/mise.json b/schema/mise.json index b7864ae5a..1c7c6cb6e 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -508,6 +508,10 @@ "description": "Specify the architecture to use for precompiled binaries.", "type": "string" }, + "precompiled_flavor": { + "description": "Specify the flavor to use for precompiled binaries.", + "type": "string" + }, "precompiled_os": { "description": "Specify the OS to use for precompiled binaries.", "type": "string" diff --git a/settings.toml b/settings.toml index bf953b3ac..829f528b6 100644 --- a/settings.toml +++ b/settings.toml @@ -641,6 +641,16 @@ optional = true description = "Specify the architecture to use for precompiled binaries." default_docs = '"apple-darwin" | "unknown-linux-gnu" | "unknown-linux-musl"' +[python.precompiled_flavor] +env = "MISE_PYTHON_PRECOMPILED_FLAVOR" +type = "String" +optional = true +default_docs = "install_only_stripped" +description = "Specify the flavor to use for precompiled binaries." +docs = """ +Options are available here: +""" + [python.precompiled_os] env = "MISE_PYTHON_PRECOMPILED_OS" type = "String" diff --git a/src/file.rs b/src/file.rs index eaf343f2b..2ebf2cfcc 100644 --- a/src/file.rs +++ b/src/file.rs @@ -594,12 +594,15 @@ pub fn un_bz2(input: &Path, dest: &Path) -> Result<()> { #[derive(Default, Clone, Copy, strum::EnumString, strum::Display)] pub enum TarFormat { #[default] + Auto, #[strum(serialize = "tar.gz")] TarGz, #[strum(serialize = "tar.xz")] TarXz, #[strum(serialize = "tar.bz2")] TarBz2, + #[strum(serialize = "tar.zst")] + TarZst, } #[derive(Default)] @@ -617,12 +620,7 @@ pub fn untar(archive: &Path, dest: &Path, opts: &TarOptions) -> Result<()> { archive.file_name().unwrap().to_string_lossy() )); } - let f = File::open(archive)?; - let tar: Box = match opts.format { - TarFormat::TarGz => Box::new(GzDecoder::new(f)), - TarFormat::TarXz => Box::new(xz2::read::XzDecoder::new(f)), - TarFormat::TarBz2 => Box::new(bzip2::read::BzDecoder::new(f)), - }; + let tar = open_tar(opts.format, archive)?; let err = || { let archive = display_path(archive); let dest = display_path(dest); @@ -666,6 +664,22 @@ pub fn untar(archive: &Path, dest: &Path, opts: &TarOptions) -> Result<()> { Ok(()) } +fn open_tar(format: TarFormat, archive: &Path) -> Result> { + let f = File::open(archive)?; + Ok(match format { + TarFormat::TarGz => Box::new(GzDecoder::new(f)), + TarFormat::TarXz => Box::new(xz2::read::XzDecoder::new(f)), + TarFormat::TarBz2 => Box::new(bzip2::read::BzDecoder::new(f)), + TarFormat::TarZst => Box::new(zstd::stream::read::Decoder::new(f)?), + TarFormat::Auto => match archive.extension().and_then(|s| s.to_str()) { + Some("xz") => open_tar(TarFormat::TarXz, archive)?, + Some("bz2") => open_tar(TarFormat::TarBz2, archive)?, + Some("zst") => open_tar(TarFormat::TarZst, archive)?, + _ => open_tar(TarFormat::TarGz, archive)?, + }, + }) +} + pub fn unzip(archive: &Path, dest: &Path) -> Result<()> { // TODO: show progress debug!("unzip {} -d {}", archive.display(), dest.display()); diff --git a/src/plugins/core/python.rs b/src/plugins/core/python.rs index 9d9880cbd..adc723d8f 100644 --- a/src/plugins/core/python.rs +++ b/src/plugins/core/python.rs @@ -4,7 +4,7 @@ use crate::cache::{CacheManager, CacheManagerBuilder}; use crate::cli::args::BackendArg; use crate::cmd::CmdLineRunner; use crate::config::{Config, SETTINGS}; -use crate::file::{display_path, TarFormat, TarOptions}; +use crate::file::{display_path, TarOptions}; use crate::git::Git; use crate::http::{HTTP, HTTP_FETCH}; use crate::install_context::InstallContext; @@ -33,6 +33,7 @@ impl PythonPlugin { ba.cache_path.join("precompiled.msgpack.z"), ) .with_fresh_duration(SETTINGS.fetch_remote_versions_cache()) + .with_cache_key(python_precompiled_platform()) .build(), ba, } @@ -92,9 +93,7 @@ impl PythonPlugin { // using http is not a security concern and enabling tls makes mise significantly slower false => HTTP_FETCH.get_text("http://mise-versions.jdx.dev/python-precompiled"), }?; - let arch = python_arch(); - let os = python_os(); - let platform = format!("{arch}-{os}"); + let platform = python_precompiled_platform(); // order by version, whether it is a release candidate, date, and in the preferred order of install types let rank = |v: &str, date: &str, name: &str| { let rc = if regex!(r"rc\d+$").is_match(v) { 0 } else { 1 }; @@ -149,12 +148,8 @@ impl PythonPlugin { "mise settings python.compile=1" ); } - let arch = python_arch(); - let os = python_os(); - bail!( - "no precompiled python found for {} on {arch}-{os}", - tv.version - ); + let platform = python_precompiled_platform(); + bail!("no precompiled python found for {tv} on {platform}"); } let available = precompiled_versions.iter().map(|(v, _, _)| v).collect_vec(); if available.is_empty() { @@ -195,11 +190,19 @@ impl PythonPlugin { &tarball_path, &install, &TarOptions { - format: TarFormat::TarGz, strip_components: 1, pr: Some(ctx.pr.as_ref()), + ..Default::default() }, )?; + if !install.join("bin").exists() { + // debug builds of indygreg binaries have a different structure + for entry in file::ls(&install.join("install"))? { + let filename = entry.file_name().unwrap(); + file::remove_all(install.join(filename))?; + file::rename(&entry, install.join(filename))?; + } + } #[cfg(unix)] file::make_symlink(&install.join("bin/python3"), &install.join("bin/python"))?; Ok(()) @@ -478,6 +481,16 @@ fn python_arch() -> &'static str { } } +fn python_precompiled_platform() -> String { + let os = python_os(); + let arch = python_arch(); + if let Some(flavor) = &SETTINGS.python.precompiled_flavor { + format!("{arch}-{os}-{flavor}") + } else { + format!("{arch}-{os}") + } +} + fn ensure_not_windows() -> eyre::Result<()> { if cfg!(windows) { bail!("python can not currently be compiled on windows with core:python, use vfox:python instead"); diff --git a/tasks.md b/tasks.md index 82283b5ac..62002f263 100644 --- a/tasks.md +++ b/tasks.md @@ -110,6 +110,10 @@ User to run as - **Usage**: `filetask.bat` +## `install-dev` + +- **Usage**: `install-dev` + ## `lint` - Depends: lint:*