diff --git a/CHANGES.md b/CHANGES.md index c0bdb64..f5a6f89 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Release Notes +## 0.2.0 + +Add support for injecting `--layout packed` PEXes. + ## 0.1.0 Initial release. diff --git a/Cargo.lock b/Cargo.lock index 86af8aa..e5c9e22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,6 +199,32 @@ dependencies = [ "serde", ] +[[package]] +name = "build-system" +version = "0.0.0" +dependencies = [ + "anyhow", + "bstr", + "const_format", + "dirs", + "flate2", + "fs-err", + "hex", + "itertools 0.14.0", + "reqwest", + "serde", + "sha2", + "strum", + "strum_macros", + "tar", + "target-lexicon", + "tempfile", + "toml", + "which", + "xz2", + "zip", +] + [[package]] name = "build-target" version = "0.8.0" @@ -1128,9 +1154,9 @@ dependencies = [ "pep440_rs", "pep508_rs", "platform", - "resources", "rstest", "same-file", + "scripts", "serde", "serde_json", "target-lexicon", @@ -1466,6 +1492,7 @@ version = "0.0.0" dependencies = [ "anstream 1.0.0", "anyhow", + "build-system", "cache", "clap", "clap-verbosity-flag", @@ -1473,7 +1500,6 @@ dependencies = [ "env_logger", "fs-err", "owo-colors", - "pexrc-build-system", "platform", "sha2", ] @@ -1541,10 +1567,12 @@ dependencies = [ "pep508_rs", "python-pkginfo", "rayon", - "resources", "rstest", + "scripts", "serde", "serde_json", + "strum", + "strum_macros", "tempfile", "testing", "url", @@ -1553,12 +1581,13 @@ dependencies = [ [[package]] name = "pexrc" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anstream 1.0.0", "anyhow", "boot", "bstr", + "build-system", "cache", "clap", "clap-verbosity-flag", @@ -1572,49 +1601,24 @@ dependencies = [ "log", "owo-colors", "pex", - "pexrc-build-system", "pexrs", "platform", - "resources", + "scripts", "sha2", "strum", "tempfile", + "walkdir", "zip", "zstd", ] -[[package]] -name = "pexrc-build-system" -version = "0.0.0" -dependencies = [ - "anyhow", - "bstr", - "const_format", - "dirs", - "flate2", - "fs-err", - "hex", - "itertools 0.14.0", - "reqwest", - "serde", - "sha2", - "strum", - "strum_macros", - "tar", - "target-lexicon", - "tempfile", - "toml", - "which", - "xz2", - "zip", -] - [[package]] name = "pexrs" version = "0.0.0" dependencies = [ "anyhow", "cache", + "fs-err", "interpreter", "itertools 0.14.0", "log", @@ -1996,18 +2000,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "resources" -version = "0.0.0" -dependencies = [ - "anyhow", - "env_logger", - "pexrc-build-system", - "strum", - "strum_macros", - "zip", -] - [[package]] name = "rfc2047-decoder" version = "1.1.0" @@ -2192,6 +2184,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scripts" +version = "0.0.0" +dependencies = [ + "anyhow", + "build-system", + "env_logger", + "fs-err", + "strum", + "strum_macros", + "zip", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2477,8 +2482,8 @@ dependencies = [ "cache", "ctor", "fs-err", - "resources", "rstest", + "scripts", "target-lexicon", "tempfile", ] @@ -2836,11 +2841,12 @@ dependencies = [ "pex", "platform", "rayon", - "resources", "rstest", + "scripts", "target-lexicon", "tempfile", "testing", + "walkdir", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 6f4ee8f..3c36b74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,12 @@ # Copyright 2026 Pex project contributors. # SPDX-License-Identifier: Apache-2.0 +[package] +name = "pexrc" +version = "0.2.0" +edition = "2024" +publish = false + [workspace] members = [ ".", @@ -10,20 +16,14 @@ members = [ "crates/interpreter", "crates/package", "crates/pex", - "crates/pexrc-build-system", + "crates/build-system", "crates/pexrs", "crates/platform", - "crates/resources", + "crates/scripts", "crates/testing", "crates/venv", ] -[package] -name = "pexrc" -version = "0.1.0" -edition = "2024" -publish = false - [package.metadata.build] zig_version = "0.15.2" @@ -140,6 +140,7 @@ textwrap = "0.16" time = "0.3" toml = "1.0" url = "2.5" +walkdir = "2.5" which = "8.0" xz2 = "0.1" zip = { version = "8.1", default-features = false, features = ["deflate", "zstd"] } @@ -147,9 +148,9 @@ zstd = "0.13" [build-dependencies] anyhow = { workspace = true } +build-system = { path = "crates/build-system" } bstr = { workspace = true } fs-err = { workspace = true } -pexrc-build-system = { path = "crates/pexrc-build-system" } env_logger = { workspace = true } itertools = { workspace = true } zstd = { workspace = true } @@ -173,9 +174,10 @@ owo-colors = { workspace = true } pex = { path = "crates/pex" } pexrs = { path = "crates/pexrs" } platform = { path = "crates/platform" } -resources = { path = "crates/resources", features = ["embedded"] } +scripts = { path = "crates/scripts", features = ["embedded"] } sha2 = { workspace = true } strum = { workspace = true } tempfile = { workspace = true } +walkdir = { workspace = true } zip = { workspace = true } zstd = { workspace = true } diff --git a/build.rs b/build.rs index f2584be..742320d 100644 --- a/build.rs +++ b/build.rs @@ -4,13 +4,11 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::{env, fs, io, iter}; +use std::{env, io, iter}; use anyhow::{anyhow, bail}; use bstr::ByteSlice; -use fs_err::File; -use itertools::Itertools; -use pexrc_build_system::{ +use build_system::{ ClassifiedTargets, ClibConfiguration, FoundTool, @@ -18,6 +16,9 @@ use pexrc_build_system::{ classify_targets, ensure_tools_installed, }; +use fs_err as fs; +use fs_err::File; +use itertools::Itertools; fn main() -> anyhow::Result<()> { println!("cargo::rerun-if-changed=crates"); diff --git a/crates/boot/src/boot.py b/crates/boot/src/boot.py index 6e00b95..a603c40 100644 --- a/crates/boot/src/boot.py +++ b/crates/boot/src/boot.py @@ -1,6 +1,6 @@ +# -*- coding: utf-8 -*- # Copyright 2026 Pex project contributors. # SPDX-License-Identifier: Apache-2.0 -# -*- coding: utf-8 -*- from __future__ import print_function @@ -437,22 +437,38 @@ def boot( def _load_pexrc(): # type: () -> Pexrc - dll = None # type: Optional[Pexrc] + prefix = ".clib" if __name__ == "__pex__" else os.path.join("__pex__", ".clib") target_triple = CURRENT_OS.target_triple(arch=CURRENT_ARCH, abi=CURRENT_ABI) library_file_name = CURRENT_OS.library_file_name(lib_name="pexrc") + library_file_relpath = os.path.join( + prefix, + "{target_triple}.{library_file_name}".format( + target_triple=target_triple, library_file_name=library_file_name + ), + ) + if __file__ and os.path.isfile(__file__): + # We're in a either a loose or packed PEX. + library_file_path = os.path.join(os.path.dirname(__file__), library_file_relpath) + if not os.path.exists(library_file_path): + raise RuntimeError( + "Pexrc is not supported on {target_triple}: no pexrc library found.".format( + target_triple=target_triple, + ) + ) + try: + return cdll.LoadLibrary(library_file_path) # type: Pexrc + except OSError as e: + raise RuntimeError( + "Failed to load pexrc library from {library_file_path}: {err}".format( + library_file_path=library_file_path, err=e + ) + ) + + dll = None # type: Optional[Pexrc] tmp_dir = tempfile.mkdtemp() - library_file_path = os.path.join(tmp_dir, os.path.basename(library_file_name)) + library_file_path = os.path.join(tmp_dir, library_file_name) try: - prefix = ".clib" if __name__ == "__pex__" else os.path.join("__pex__", ".clib") - pexrc_data = pkgutil.get_data( - __name__, - os.path.join( - prefix, - "{target_triple}.{library_file_name}".format( - target_triple=target_triple, library_file_name=library_file_name - ), - ), - ) + pexrc_data = pkgutil.get_data(__name__, library_file_relpath) if pexrc_data is None: raise RuntimeError( "Pexrc is not supported on {target_triple}: no pexrc library found.".format( diff --git a/crates/boot/src/boot.sh b/crates/boot/src/boot.sh index dba9d88..f12e1b8 100644 --- a/crates/boot/src/boot.sh +++ b/crates/boot/src/boot.sh @@ -1,15 +1,19 @@ #!/bin/sh +# -*- coding: utf-8 -*- +# Copyright 2026 Pex project contributors. +# SPDX-License-Identifier: Apache-2.0 +# --- split --- # # N.B.: This script should stick to syntax defined for POSIX `sh` and avoid non-builtins. # See: https://pubs.opengroup.org/onlinepubs/9699919799/idx/shell.html set -eu -# --- vars --- # +# --- split --- # # N.B.: These vars are templated in by pexrc when it injects a PEX with its runtime. RAW_DEFAULT_PEXRC_ROOT="{pexrc_root}" VENV_RELPATH="{venv_relpath}" PYTHONS="{pythons}" -# --- vars --- # +# --- split --- # # N.B.: The SC2116 warning suppressions below are in place to ensure tilde-expansion of the # DEFAULT_PEX_ROOT value which is necessary for the -x check of the venv pex to succeed when it diff --git a/crates/boot/src/lib.rs b/crates/boot/src/lib.rs index b96fa30..da99d10 100644 --- a/crates/boot/src/lib.rs +++ b/crates/boot/src/lib.rs @@ -7,6 +7,7 @@ use std::path::Path; use anyhow::{anyhow, bail}; use cache::CacheDir; use const_format::str_split; +use fs_err as fs; use fs_err::File; use interpreter::{InterpreterConstraints, SearchPath, SelectionStrategy}; use pex::{Pex, PexPath}; @@ -16,11 +17,13 @@ use zip::ZipWriter; use zip::write::{FileOptionExtension, FileOptions}; const SH_BOOT_SHEBANG: &[u8] = b"#!/bin/sh\n"; -const SH_BOOT_PARTS: [&str; 3] = str_split!(include_str!("boot.sh"), "# --- vars --- #\n"); +const SH_BOOT_PARTS: [&str; 4] = str_split!(include_str!("boot.sh"), "# --- split --- #\n"); + +pub fn sh_boot_shebang(pex: &Path, escaped: bool) -> anyhow::Result> { + let pex = Pex::load(pex)?; -pub fn sh_boot_shebang(pex: &Path) -> anyhow::Result> { let mut sh_boot_shebang_buffer: [_; SH_BOOT_SHEBANG.len()] = [0; SH_BOOT_SHEBANG.len()]; - let mut pex_fp = File::open(pex)?; + let mut pex_fp = File::open(pex.file())?; match pex_fp.read_exact(&mut sh_boot_shebang_buffer) { Ok(()) => { if sh_boot_shebang_buffer != SH_BOOT_SHEBANG { @@ -30,21 +33,18 @@ pub fn sh_boot_shebang(pex: &Path) -> anyhow::Result> { Err(err) if err.kind() == ErrorKind::UnexpectedEof => return Ok(None), Err(err) => bail!( "Failed to determine if {pex} uses a `--sh-boot` shebang header: {err}", - pex = pex.display() + pex = pex.path.display() ), }; - - let pex = Pex::load(pex)?; - let pex_info = pex.info(); - let pex_path = PexPath::from_pex_info(pex_info, false); + let pex_path = PexPath::from_pex_info(&pex.info, false); let additional_pexes = pex_path.load_pexes()?; let venv_dir = venv_dir(None, &pex, &SearchPath::EMPTY, &additional_pexes)?; let venv_relpath = venv_dir.strip_prefix(CacheDir::root()?)?; let interpreter_constraints = - InterpreterConstraints::try_from(&pex_info.interpreter_constraints)?; - let selection_strategy: SelectionStrategy = pex_info.interpreter_selection_strategy.into(); + InterpreterConstraints::try_from(&pex.info.interpreter_constraints)?; + let selection_strategy: SelectionStrategy = pex.info.interpreter_selection_strategy.into(); let pythons = interpreter_constraints .calculate_compatible_binary_names(selection_strategy) .into_iter() @@ -56,16 +56,19 @@ pub fn sh_boot_shebang(pex: &Path) -> anyhow::Result> { .collect::>>()?; Ok(Some(format!( - "{header}{vars}{body}\n", - header = SH_BOOT_PARTS[0], - vars = SH_BOOT_PARTS[1] + "{shebang}{start_escape}{header}{vars}{body}{end_escape}\n", + shebang = SH_BOOT_PARTS[0], // N.B.: SH_BOOT_SHEBANG + start_escape = if escaped { "'''': pshprs\n" } else { "" }, + header = SH_BOOT_PARTS[1], + vars = SH_BOOT_PARTS[2] .replace( "{pexrc_root}", - pex_info.pex_root.as_deref().unwrap_or_default() + pex.info.pex_root.as_deref().unwrap_or_default(), ) .replace("{venv_relpath}", path_as_str(venv_relpath)?) .replace("{pythons}", &pythons.join("\n")), - body = SH_BOOT_PARTS[2] + body = SH_BOOT_PARTS[3].trim_end(), + end_escape = if escaped { "\n'''\n" } else { "\n" }, ))) } @@ -81,3 +84,15 @@ pub fn inject_boot( zip.write_all(PY_BOOT)?; Ok(()) } + +pub fn write_boot(dest_dir: &Path, shebang: &str) -> anyhow::Result<()> { + let main_py_path = dest_dir.join("__main__.py"); + let mut file = File::create_new(&main_py_path)?; + file.write_all(shebang.as_bytes().trim_ascii_end())?; + file.write_all(b"\n\n")?; + file.write_all(PY_BOOT)?; + fs::copy(&main_py_path, dest_dir.join("__pex__").join("__init__.py"))?; + platform::mark_executable(file.file_mut())?; + platform::symlink_or_link_or_copy(&main_py_path, dest_dir.join("pex"), true)?; + Ok(()) +} diff --git a/crates/pexrc-build-system/Cargo.toml b/crates/build-system/Cargo.toml similarity index 96% rename from crates/pexrc-build-system/Cargo.toml rename to crates/build-system/Cargo.toml index 2d17bec..9090769 100644 --- a/crates/pexrc-build-system/Cargo.toml +++ b/crates/build-system/Cargo.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [package] -name = "pexrc-build-system" +name = "build-system" edition = "2024" [dependencies] diff --git a/crates/pexrc-build-system/src/downloads.rs b/crates/build-system/src/downloads.rs similarity index 99% rename from crates/pexrc-build-system/src/downloads.rs rename to crates/build-system/src/downloads.rs index ebe2961..3fb6b90 100644 --- a/crates/pexrc-build-system/src/downloads.rs +++ b/crates/build-system/src/downloads.rs @@ -1,11 +1,12 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 +use std::io; use std::io::{ErrorKind, Read}; use std::path::{Path, PathBuf}; -use std::{fs, io}; use anyhow::{anyhow, bail}; +use fs_err as fs; use fs_err::File; use sha2::{Digest, Sha256}; diff --git a/crates/pexrc-build-system/src/lib.rs b/crates/build-system/src/lib.rs similarity index 100% rename from crates/pexrc-build-system/src/lib.rs rename to crates/build-system/src/lib.rs diff --git a/crates/pexrc-build-system/src/metadata.rs b/crates/build-system/src/metadata.rs similarity index 100% rename from crates/pexrc-build-system/src/metadata.rs rename to crates/build-system/src/metadata.rs diff --git a/crates/pexrc-build-system/src/rust_toolchain.rs b/crates/build-system/src/rust_toolchain.rs similarity index 100% rename from crates/pexrc-build-system/src/rust_toolchain.rs rename to crates/build-system/src/rust_toolchain.rs diff --git a/crates/pexrc-build-system/src/tools.rs b/crates/build-system/src/tools.rs similarity index 99% rename from crates/pexrc-build-system/src/tools.rs rename to crates/build-system/src/tools.rs index 33a9b1e..e3d2623 100644 --- a/crates/pexrc-build-system/src/tools.rs +++ b/crates/build-system/src/tools.rs @@ -2,14 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; +use std::env; use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::{env, fs}; use anyhow::bail; use bstr::ByteSlice; use const_format::concatcp; +use fs_err as fs; use fs_err::File; use strum::{EnumCount, IntoEnumIterator}; use strum_macros::{EnumCount, EnumIter}; diff --git a/crates/interpreter/Cargo.toml b/crates/interpreter/Cargo.toml index 82a2416..583e73c 100644 --- a/crates/interpreter/Cargo.toml +++ b/crates/interpreter/Cargo.toml @@ -16,8 +16,8 @@ fs-err = { workspace = true } log = { workspace = true } logging_timer = { workspace = true } platform = { path = "../platform" } -resources = { path = "../resources" } same-file = { workspace = true } +scripts = { path = "../scripts" } serde = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } @@ -28,8 +28,8 @@ which = { workspace = true } [dev-dependencies] anyhow = { workspace = true } pep440_rs = { workspace = true } -resources = { path = "../resources" } rstest = { workspace = true } +scripts = { path = "../scripts" } target-lexicon = { workspace = true } testing = { path = "../testing" } textwrap = { workspace = true } \ No newline at end of file diff --git a/crates/interpreter/src/interpreter.rs b/crates/interpreter/src/interpreter.rs index 676261c..a7b4b43 100644 --- a/crates/interpreter/src/interpreter.rs +++ b/crates/interpreter/src/interpreter.rs @@ -6,15 +6,13 @@ use std::fmt::{Display, Formatter}; use std::io::{BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::Mutex; use anyhow::{anyhow, bail}; use cache::{CacheDir, HashOptions, atomic_file, hash_file}; use fs_err as fs; -use log::debug; use logging_timer::time; use pep508_rs::MarkerEnvironment; -use resources::{InterpreterIdentificationScript, Resources}; +use scripts::{IdentifyInterpreter, Scripts}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] @@ -79,12 +77,12 @@ pub struct Interpreter { } #[cfg(target_os = "linux")] -static LINUX_INFO: Mutex> = Mutex::new(None); +static LINUX_INFO: std::sync::Mutex> = std::sync::Mutex::new(None); impl Interpreter { fn identify( python_exe: impl AsRef, - identification_script: &InterpreterIdentificationScript, + identification_script: &IdentifyInterpreter, ) -> anyhow::Result> { let mut script = tempfile::Builder::new() .prefix("virtualenv.") @@ -95,6 +93,8 @@ impl Interpreter { command.arg("-sE").arg(script.path()); #[cfg(target_os = "linux")] { + use log::debug; + let mut linux_info = LINUX_INFO .lock() .map_err(|err| anyhow!("Failed to obtain lock on Linux platform info: {err}"))?; @@ -131,7 +131,7 @@ impl Interpreter { pub fn load_uncached( python_exe: impl AsRef, - identification_script: &InterpreterIdentificationScript, + identification_script: &IdentifyInterpreter, ) -> anyhow::Result { let json_bytes = Self::identify(python_exe.as_ref(), identification_script)?; serde_json::from_slice(&json_bytes).map_err(|err| { @@ -226,11 +226,11 @@ impl Interpreter { } } - fn at_prefix<'a>( + fn at_prefix( prefix: impl AsRef, version: PythonVersion, pypy_version: Option, - resources: &mut impl Resources<'a>, + scripts: &mut Scripts, re_cache_version_mismatch: bool, ) -> anyhow::Result { let check_pypy_version = |interpreter: &Interpreter| match ( @@ -245,7 +245,7 @@ impl Interpreter { (None, None) => true, _ => false, }; - let identification_script = InterpreterIdentificationScript::read(resources)?; + let identification_script = IdentifyInterpreter::read(scripts)?; let candidate_rel_paths = Self::candidate_rel_paths(&version, pypy_version.as_ref()); let mut re_cache_candidates: Vec = Vec::with_capacity(candidate_rel_paths.len()); for rel_path in candidate_rel_paths { @@ -292,7 +292,7 @@ impl Interpreter { #[time("debug", "Interpreter.{}")] pub fn load( python_exe: impl AsRef, - identification_script: &InterpreterIdentificationScript, + identification_script: &IdentifyInterpreter, ) -> anyhow::Result { let interpreter_info = Self::interpreter_info(python_exe.as_ref())?; Self::load_internal(interpreter_info, python_exe, identification_script) @@ -301,7 +301,7 @@ impl Interpreter { fn load_internal( interpreter_info: impl AsRef, python_exe: impl AsRef, - identification_script: &InterpreterIdentificationScript, + identification_script: &IdentifyInterpreter, ) -> anyhow::Result { let file = atomic_file(interpreter_info.as_ref(), |file| { let json_bytes = Self::identify(python_exe.as_ref(), identification_script)?; @@ -316,10 +316,7 @@ impl Interpreter { }) } - fn reload( - self, - identification_script: &InterpreterIdentificationScript, - ) -> anyhow::Result { + fn reload(self, identification_script: &IdentifyInterpreter) -> anyhow::Result { let interpreter_info = Self::interpreter_info(self.path.as_path())?; fs::remove_file(&interpreter_info)?; Self::load_internal(&interpreter_info, self.path, identification_script) @@ -347,19 +344,14 @@ impl Interpreter { #[time("debug", "Interpreter.{}")] pub fn resolve_base_interpreter<'a>( self, - resources: &mut impl Resources<'a>, + scripts: &mut Scripts, ) -> anyhow::Result { if let Some(base_prefix) = self.base_prefix.as_ref() && base_prefix != &self.prefix { - let resolved = Self::at_prefix( - base_prefix, - self.version, - self.pypy_version, - resources, - true, - )?; - return resolved.resolve_base_interpreter(resources); + let resolved = + Self::at_prefix(base_prefix, self.version, self.pypy_version, scripts, true)?; + return resolved.resolve_base_interpreter(scripts); } Ok(self) } @@ -372,10 +364,10 @@ mod tests { use std::process::{Command, Stdio}; use anyhow::Context; - use resources::{InterpreterIdentificationScript, Resources}; use rstest::rstest; + use scripts::{IdentifyInterpreter, Scripts}; use testing::{ - embedded_resources, + embedded_scripts, interpreter_identification_script, python_exe, venv_python_exe, @@ -387,7 +379,7 @@ mod tests { #[rstest] fn test_tags_same_as_packaging( venv_python_exe: PathBuf, - interpreter_identification_script: InterpreterIdentificationScript, + interpreter_identification_script: IdentifyInterpreter, ) { assert!( Command::new(&venv_python_exe) @@ -435,10 +427,9 @@ mod tests { fn test_resolve_base_interpreter( python_exe: &Path, venv_python_exe: PathBuf, - mut embedded_resources: impl Resources<'static>, + mut embedded_scripts: Scripts, ) { - let identification_script = - InterpreterIdentificationScript::read(&mut embedded_resources).unwrap(); + let identification_script = IdentifyInterpreter::read(&mut embedded_scripts).unwrap(); let venv_interpreter = Interpreter::load(&venv_python_exe, &identification_script) .with_context(|| { format!( @@ -450,7 +441,7 @@ mod tests { assert_eq!( python_exe, venv_interpreter - .resolve_base_interpreter(&mut embedded_resources) + .resolve_base_interpreter(&mut embedded_scripts) .unwrap() .path ) diff --git a/crates/package/Cargo.toml b/crates/package/Cargo.toml index 406887e..ba3327f 100644 --- a/crates/package/Cargo.toml +++ b/crates/package/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] anstream = { workspace = true } anyhow = { workspace = true } +build-system = { path = "../build-system" } cache = { path = "../cache" } clap = { workspace = true } clap-verbosity-flag = { workspace = true } @@ -12,6 +13,5 @@ colorchoice-clap = { workspace = true } env_logger = { workspace = true } fs-err = { workspace = true } owo-colors = { workspace = true } -pexrc-build-system = { path = "../pexrc-build-system" } platform = { path = "../platform" } sha2 = { workspace = true } diff --git a/crates/package/src/main.rs b/crates/package/src/main.rs index 22a9e24..222e8cc 100644 --- a/crates/package/src/main.rs +++ b/crates/package/src/main.rs @@ -9,13 +9,13 @@ use std::sync::LazyLock; use std::{cmp, env, io}; use anyhow::{anyhow, bail}; +use build_system::{Target, all_targets, classify_targets, ensure_tools_installed}; use cache::Fingerprint; use clap::builder::Str; use clap::{ArgAction, Parser}; use fs_err as fs; use fs_err::File; use owo_colors::OwoColorize; -use pexrc_build_system::{Target, all_targets, classify_targets, ensure_tools_installed}; use sha2::{Digest, Sha256}; static CARGO: LazyLock = LazyLock::new(|| env!("CARGO").into()); diff --git a/crates/pex/Cargo.toml b/crates/pex/Cargo.toml index 1a7b430..0fa2b0b 100644 --- a/crates/pex/Cargo.toml +++ b/crates/pex/Cargo.toml @@ -15,22 +15,24 @@ log = { workspace = true } logging_timer = { workspace = true } pep440_rs = { workspace = true } pep508_rs = { workspace = true } -resources = { path = "../resources" } python-pkginfo = { workspace = true } rayon = { workspace = true } +scripts = { path = "../scripts" } serde = { workspace = true } serde_json = { workspace = true } +strum = { workspace = true } +strum_macros = { workspace = true } url = { workspace = true } zip = { workspace = true } [dev-dependencies] +fs-err = { workspace = true } glob = { workspace = true } indexmap = { workspace = true } interpreter = { path = "../interpreter" } -pep440_rs = { workspace = true } pep508_rs = { workspace = true } -resources = { path = "../resources" } rstest = { workspace = true } +scripts = { path = "../scripts" } tempfile = { workspace = true } testing = { path = "../testing" } url = { workspace = true } diff --git a/crates/pex/src/lib.rs b/crates/pex/src/lib.rs index 132583e..a132395 100644 --- a/crates/pex/src/lib.rs +++ b/crates/pex/src/lib.rs @@ -6,6 +6,6 @@ mod pex_info; mod pex_path; mod wheel; -pub use pex::{LoosePex, PackedPex, Pex, WheelResolver, ZipAppPex}; +pub use pex::{Layout, Pex}; pub use pex_info::{BinPath, InheritPath, InterpreterSelectionStrategy, PexInfo}; pub use pex_path::PexPath; diff --git a/crates/pex/src/pex.rs b/crates/pex/src/pex.rs index d34ebd1..fdc3622 100644 --- a/crates/pex/src/pex.rs +++ b/crates/pex/src/pex.rs @@ -3,13 +3,14 @@ use std::borrow::Cow; use std::collections::{HashMap, VecDeque}; -use std::io; -use std::io::{BufReader, Read, Seek}; +use std::ffi::OsStr; +use std::io::{BufReader, Read}; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, Mutex}; use anyhow::{anyhow, bail}; +use fs_err as fs; use fs_err::File; use indexmap::{IndexMap, IndexSet}; use interpreter::{Interpreter, InterpreterConstraints, SearchPath}; @@ -19,26 +20,13 @@ use logging_timer::{time, timer}; use pep440_rs::Version; use pep508_rs::{ExtraName, PackageName, Requirement, VersionOrUrl}; use rayon::prelude::*; -use resources::{InterpreterIdentificationScript, ResourcePath, Resources}; +use scripts::{IdentifyInterpreter, Scripts}; +use strum_macros::{AsRefStr, EnumString}; use url::Url; use zip::ZipArchive; use crate::PexInfo; -use crate::wheel::{MetadataReader, Tag, WheelFile, WheelMetadata}; - -pub trait WheelResolver { - fn resolve(&self, interpreter: &Interpreter) -> anyhow::Result>; -} - -pub struct LoosePex<'a>(pub &'a Path, pub PexInfo); -pub struct PackedPex<'a>(pub &'a Path, pub PexInfo); -pub struct ZipAppPex<'a>(pub &'a Path, pub PexInfo); - -impl<'a> ZipAppPex<'a> { - pub(crate) fn resources(&self) -> anyhow::Result> { - ZipResources::new(self.0) - } -} +use crate::wheel::{MetadataReader, Tag, WheelFile, WheelMetadata, WhlMetadataReader}; struct ZipAppPexMetadataReader<'a> { zip: &'a mut ZipArchive, @@ -56,16 +44,101 @@ impl<'a> MetadataReader for ZipAppPexMetadataReader<'a> { } } -// TODO: XXX: This just uses PEX-INFO to resolve wheel file names, it is not ZipAppPex-specific. -impl<'a> WheelResolver for ZipAppPex<'a> { - #[time("debug", "WheelResolver.{}")] - fn resolve(&self, interpreter: &Interpreter) -> anyhow::Result> { - let python_version = Version::new([ - u64::from(interpreter.version.major), - u64::from(interpreter.version.minor), - u64::from(interpreter.version.micro), - ]); +#[derive(AsRefStr, EnumString)] +pub enum Layout { + #[strum(serialize = "loose")] + Loose, + #[strum(serialize = "packed")] + Packed, + #[strum(serialize = "zipapp")] + ZipApp, +} + +impl Layout { + pub fn try_load(pex_dir: &Path) -> anyhow::Result { + Ok(Layout::from_str( + fs::read_to_string(pex_dir.join("PEX-LAYOUT"))?.trim(), + )?) + } + + pub fn record(&self, pex_dir: &Path) -> anyhow::Result<()> { + fs::write(pex_dir.join("PEX-LAYOUT"), self.as_ref())?; + Ok(()) + } +} + +pub struct Pex<'a> { + pub path: &'a Path, + pub info: PexInfo, + pub layout: Layout, +} + +impl<'a> Pex<'a> { + #[time("debug", "Pex.{}")] + pub fn load(path: &'a Path) -> anyhow::Result { + if path.is_file() { + let zip_fp = File::open(path)?; + let mut zip = { + let _timer = timer!(Level::Debug; "Open PEX zip", "{}", path.display()); + ZipArchive::new(BufReader::new(zip_fp))? + }; + let pex_info = + PexInfo::parse(zip.by_name("PEX-INFO")?, Some(|| Cow::Borrowed("PEX-INFO")))?; + Ok(Self { + path, + info: pex_info, + layout: Layout::ZipApp, + }) + } else { + let layout = match Layout::try_load(path) { + Ok(layout) => layout, + _ => { + let deps_dir = path.join(".deps"); + if deps_dir.is_dir() + && let Some(wheel) = fs::read_dir(&deps_dir)? + .filter_map(|entry| { + entry + .ok() + .filter(|e| e.path().extension() == Some(OsStr::new("whl"))) + }) + .next() + && wheel.path().is_file() + { + Layout::Packed + } else { + Layout::Loose + } + } + }; + let pex_info_path = path.join("PEX-INFO"); + let pex_info_fp = File::open(&pex_info_path)?; + let pex_info = PexInfo::parse(pex_info_fp, Some(|| pex_info_path.to_string_lossy()))?; + Ok(Self { + path, + info: pex_info, + layout, + }) + } + } + + pub fn file(&self) -> Cow<'a, Path> { + match self.layout { + Layout::Loose | Layout::Packed => Cow::Owned(self.path.join("pex")), + Layout::ZipApp => Cow::Borrowed(self.path), + } + } + + pub fn resources(&self) -> anyhow::Result { + let path = self.path.to_path_buf(); + match self.layout { + Layout::Packed | Layout::Loose => Ok(Scripts::Loose(path)), + Layout::ZipApp => Ok(Scripts::Zipped(ZipArchive::new(File::open(&path)?)?)), + } + } + + #[time("debug", "Pex.{}")] + fn resolve_wheels(&self, interpreter: &Interpreter) -> anyhow::Result> { let supported_tags: HashMap = interpreter .supported_tags .iter() @@ -74,7 +147,7 @@ impl<'a> WheelResolver for ZipAppPex<'a> { .collect::>()?; let wheel_files = self - .1 + .info .parse_distributions() .collect::, _>>()?; @@ -90,23 +163,7 @@ impl<'a> WheelResolver for ZipAppPex<'a> { }) .collect::>(); - let mut wheels = Vec::with_capacity(wheel_files.len()); - let mut zip = ZipArchive::new(File::open(self.0)?)?; - for (file_name, wheel_file, rank) in wheel_files { - let wheel = WheelMetadata::parse( - wheel_file, - ZipAppPexMetadataReader { - zip: &mut zip, - wheel_file_name: file_name, - }, - )?; - if let Some(requires_python) = &wheel.requires_python - && !requires_python.contains(&python_version) - { - continue; - } - wheels.push((file_name, wheel, rank)); - } + let wheels = self.load_wheel_metadata(interpreter, wheel_files)?; struct WheelInfo<'b>(&'b str, Version, Vec>, usize); @@ -131,7 +188,7 @@ impl<'a> WheelResolver for ZipAppPex<'a> { IndexMap::with_capacity(wheels_by_project_name.len()); let mut indexed_extras: Vec> = vec![Vec::new()]; let mut to_resolve: VecDeque<(Requirement, usize)> = self - .1 + .info .requirements .iter() .map(|requirement| { @@ -152,7 +209,7 @@ impl<'a> WheelResolver for ZipAppPex<'a> { .remove(&requirement.name) .ok_or_else(|| { let inapplicable_wheels = self - .1 + .info .parse_distributions() .filter_map(|result| match result { Ok((file_name, wheel_file)) @@ -183,7 +240,7 @@ impl<'a> WheelResolver for ZipAppPex<'a> { "The PEX at {path} has requirement {requirement} that cannot be satisfied \ for the interpreter at {python_exe}.\n\ {reason}", - path = self.0.display(), + path = self.path.display(), python_exe = interpreter.path.display(), reason = reason, ) @@ -199,7 +256,7 @@ impl<'a> WheelResolver for ZipAppPex<'a> { VersionOrUrl::Url(url) => bail!( "A PEX should never contain an URL requirement.\ The PEX at {path} requires: {url}", - path = self.0.display() + path = self.path.display() ), } } @@ -219,98 +276,6 @@ impl<'a> WheelResolver for ZipAppPex<'a> { } Ok(resolved_by_project_name.into_values().collect()) } -} - -struct ZipResources { - zip: ZipArchive, -} - -impl ZipResources> { - fn new(path: &Path) -> anyhow::Result { - let zip = ZipArchive::new(BufReader::new(File::open(path)?))?; - Ok(Self { zip }) - } -} - -impl<'a, R: Read + Seek> Resources<'a> for ZipResources { - fn read(&mut self, path: ResourcePath) -> anyhow::Result> { - // TODO: XXX: The entry name logic here is shared with pexrc - centralize. - let entry = self - .zip - .by_name(format!("__pex__/.scripts/{script}", script = path.script_name()).as_str())?; - Ok(Cow::Owned(io::read_to_string(entry)?)) - } -} - -pub enum Pex<'a> { - Loose(LoosePex<'a>), - Packed(PackedPex<'a>), - ZipApp(ZipAppPex<'a>), -} - -impl<'a> Pex<'a> { - #[time("debug", "Pex.{}")] - pub fn load(path: &'a Path) -> anyhow::Result { - if path.is_file() { - let zip_fp = File::open(path)?; - let mut zip = { - let _timer = timer!(Level::Debug; "Open PEX zip", "{}", path.display()); - ZipArchive::new(BufReader::new(zip_fp))? - }; - let pex_info = - PexInfo::parse(zip.by_name("PEX-INFO")?, Some(|| Cow::Borrowed("PEX-INFO")))?; - Ok(Pex::ZipApp(ZipAppPex(path, pex_info))) - } else { - let bootstrap = path.join(".bootstrap"); - if !bootstrap.exists() { - bail!( - "There is no PEX at {path}: it contains no `.bootstrap`.", - path = path.display() - ) - } - let pex_info_path = path.join("PEX-INFO"); - let pex_info_fp = File::open(&pex_info_path)?; - let pex_info = PexInfo::parse(pex_info_fp, Some(|| pex_info_path.to_string_lossy()))?; - if bootstrap.is_dir() { - Ok(Pex::Loose(LoosePex(path, pex_info))) - } else { - Ok(Pex::Packed(PackedPex(path, pex_info))) - } - } - } - - pub fn path(&self) -> &Path { - match self { - Pex::Loose(pex) => pex.0, - Pex::Packed(pex) => pex.0, - Pex::ZipApp(pex) => pex.0, - } - } - - pub fn info(&self) -> &PexInfo { - match self { - Pex::Loose(pex) => &pex.1, - Pex::Packed(pex) => &pex.1, - Pex::ZipApp(pex) => &pex.1, - } - } - - pub fn resources(&self) -> anyhow::Result> { - match self { - Pex::Loose(_) => todo!("XXX: Implement loose PEX resource resolution."), - Pex::Packed(_) => todo!("XXX: Implement packed PEX resource resolution."), - Pex::ZipApp(zip_app) => zip_app.resources(), - } - } - - fn resolve_wheels(&self, interpreter: &Interpreter) -> anyhow::Result> { - let zip_app_pex = match self { - Pex::Loose(_) => todo!("XXX: Implement loose PEX wheel resolution."), - Pex::Packed(_) => todo!("XXX: Implement packed PEX wheel resolution."), - Pex::ZipApp(zip_app) => zip_app, - }; - zip_app_pex.resolve(interpreter) - } #[time("debug", "Pex.{}")] pub fn resolve( @@ -321,28 +286,21 @@ impl<'a> Pex<'a> { ) -> anyhow::Result<( Interpreter, IndexSet<&'a str>, - impl Resources<'a>, + Scripts, Vec<(&'a Pex<'a>, IndexSet<&'a str>)>, )> { - let zip_app_pex = match self { - Pex::Loose(_) => todo!("XXX: Implement loose PEX wheel resolution."), - Pex::Packed(_) => todo!("XXX: Implement packed PEX wheel resolution."), - Pex::ZipApp(zip_app) => zip_app, - }; + let mut resources = self.resources()?; + let identification_script = IdentifyInterpreter::read(&mut resources)?; - let mut resources = zip_app_pex.resources()?; - let identification_script = InterpreterIdentificationScript::read(&mut resources)?; - - let pex_info = self.info(); let interpreter_constraints = - InterpreterConstraints::try_from(&pex_info.interpreter_constraints)?; + InterpreterConstraints::try_from(&self.info.interpreter_constraints)?; let mut errors = Vec::new(); if let Some(python_exe) = python_exe && let Ok(interpreter) = Interpreter::load(python_exe, &identification_script) && interpreter_constraints.contains(&interpreter) && search_path.contains(python_exe) { - match zip_app_pex.resolve(&interpreter) { + match self.resolve_wheels(&interpreter) { Ok(selected_wheels) => { let additional_resolves = additional_pexes .map(|pex| pex.resolve_wheels(&interpreter).map(|wheels| (pex, wheels))) @@ -355,7 +313,7 @@ impl<'a> Pex<'a> { let interpreters_to_try = interpreter_constraints .iter_possibly_compatible_python_exes( - pex_info.interpreter_selection_strategy.into(), + self.info.interpreter_selection_strategy.into(), search_path, )? .collect::>(); @@ -374,7 +332,7 @@ impl<'a> Pex<'a> { }, ) .filter(|interpreter| interpreter_constraints.contains(interpreter)) - .map(|interpreter| match zip_app_pex.resolve(&interpreter) { + .map(|interpreter| match self.resolve_wheels(&interpreter) { Ok(selected_wheels) => Ok((interpreter, selected_wheels)), Err(err) => Err((interpreter, err)), }); @@ -403,7 +361,7 @@ impl<'a> Pex<'a> { return Ok((interpreter, selected_wheels, resources, additional_resolves)); } - let reqs = &self.info().requirements; + let reqs = &self.info.requirements; let requirement_count = reqs.len(); let requirements = if requirement_count == 1 { "requirement" @@ -415,7 +373,7 @@ impl<'a> Pex<'a> { anyhow!( "Failed to resolve requirements for PEX {path} and resolve errors were obfuscated \ by a poisoned lock: {err}", - path = zip_app_pex.0.display() + path = self.path.display() ) })?; let error_count = errors.len(); @@ -433,7 +391,7 @@ impl<'a> Pex<'a> { \n\ Tried resolving using {error_count} {interpreters}:\n\ {errors}", - path = self.path().display(), + path = self.path.display(), reqs = reqs.iter().map(|req| format!("+ {req}")).join("\n"), errors = errors .iter() @@ -446,6 +404,69 @@ impl<'a> Pex<'a> { .join("\n") ) } + + fn load_wheel_metadata( + &'a self, + interpreter: &Interpreter, + wheel_files: Vec<(&'a str, WheelFile<'a>, usize)>, + ) -> anyhow::Result, usize)>> { + let python_version = Version::new([ + u64::from(interpreter.version.major), + u64::from(interpreter.version.minor), + u64::from(interpreter.version.micro), + ]); + match self.layout { + Layout::Loose => todo!("Loose PEX wheel metadata reading."), + Layout::Packed => self.yyy(python_version, wheel_files), + Layout::ZipApp => self.xxx(python_version, wheel_files), + } + } + + fn xxx( + &'a self, + python_version: Version, + wheel_files: Vec<(&'a str, WheelFile<'a>, usize)>, + ) -> anyhow::Result, usize)>> { + let mut wheels = Vec::with_capacity(wheel_files.len()); + let mut zip = ZipArchive::new(File::open(self.path)?)?; + for (file_name, wheel_file, rank) in wheel_files { + let wheel = WheelMetadata::parse( + wheel_file, + ZipAppPexMetadataReader { + zip: &mut zip, + wheel_file_name: file_name, + }, + )?; + if let Some(requires_python) = &wheel.requires_python + && !requires_python.contains(&python_version) + { + continue; + } + wheels.push((file_name, wheel, rank)); + } + Ok(wheels) + } + + fn yyy( + &'a self, + python_version: Version, + wheel_files: Vec<(&'a str, WheelFile<'a>, usize)>, + ) -> anyhow::Result, usize)>> { + let mut wheels = Vec::with_capacity(wheel_files.len()); + for (file_name, wheel_file, rank) in wheel_files { + let wheel = WheelMetadata::parse( + wheel_file, + WhlMetadataReader::new(self.path.join(".deps").join(file_name))?, + )?; + if let Some(requires_python) = &wheel.requires_python + && !requires_python.contains(&python_version) + { + continue; + } + wheels.push((file_name, wheel, rank)); + } + Ok(wheels) + } } #[cfg(test)] @@ -458,15 +479,15 @@ mod tests { use indexmap::{IndexSet, indexset}; use interpreter::{Interpreter, SearchPath}; use pep508_rs::{Requirement, VersionOrUrl}; - use resources::{InterpreterIdentificationScript, Resources}; use rstest::{fixture, rstest}; - use testing::{embedded_resources, interpreter_identification_script, python_exe, tmp_dir}; + use scripts::{IdentifyInterpreter, Scripts}; + use testing::{embedded_scripts, interpreter_identification_script, python_exe, tmp_dir}; use url::Url; use zip::write::SimpleFileOptions; use zip::{CompressionMethod, ZipWriter}; use crate::wheel::WheelFile; - use crate::{Pex, PexPath, WheelResolver}; + use crate::{Pex, PexPath}; const EXPECTED_ANSICOLORS_PEX_WHEELS: [&str; 1] = ["ansicolors==1.1.8"]; @@ -502,7 +523,7 @@ mod tests { tmp_dir: PathBuf, python_exe: &Path, ansicolors_pex: PathBuf, - mut embedded_resources: impl Resources<'static>, + mut embedded_scripts: Scripts, ) -> PathBuf { let pex = tmp_dir.join("requests.pex"); assert!( @@ -526,7 +547,7 @@ mod tests { .unwrap(); let file_options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); - embedded_resources + embedded_scripts .inject_scripts(&mut zip, file_options) .unwrap(); zip.finish().unwrap(); @@ -566,23 +587,19 @@ mod tests { fn test_resolve_single( requests_pex: PathBuf, python_exe: &Path, - interpreter_identification_script: InterpreterIdentificationScript, + interpreter_identification_script: IdentifyInterpreter, ) { - let pex = match Pex::load(&requests_pex).unwrap() { - Pex::ZipApp(zip_app_pex) => zip_app_pex, - _ => panic!("Unexpected pex type"), - }; + let pex = Pex::load(&requests_pex).unwrap(); let interpreter = Interpreter::load(python_exe, &interpreter_identification_script).unwrap(); - let wheels = pex.resolve(&interpreter).unwrap(); - + let wheels = pex.resolve_wheels(&interpreter).unwrap(); assert_wheels(wheels, EXPECTED_REQUESTS_PEX_WHEELS); } #[rstest] fn test_resolve_additional(requests_pex: PathBuf, python_exe: &Path) { let pex = Pex::load(&requests_pex).unwrap(); - let pex_path = PexPath::from_pex_info(pex.info(), false); + let pex_path = PexPath::from_pex_info(&pex.info, false); let additional_pexes = pex_path.load_pexes().unwrap(); let search_path = SearchPath::known(indexset![python_exe.to_path_buf()]); let (_, wheels, _, additional_resolves) = pex diff --git a/crates/pex/src/wheel.rs b/crates/pex/src/wheel.rs index 7042e1d..c51b82a 100644 --- a/crates/pex/src/wheel.rs +++ b/crates/pex/src/wheel.rs @@ -1,7 +1,7 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 -use std::ffi::OsStr; +use std::fmt::{Display, Formatter, Write}; use std::io; use std::io::{Read, Seek}; use std::path::Path; @@ -9,6 +9,7 @@ use std::str::FromStr; use anyhow::{anyhow, bail}; use fs_err::File; +use indexmap::IndexSet; use itertools::Itertools; use pep440_rs::{Version, VersionSpecifiers}; use pep508_rs::{PackageName, Requirement}; @@ -50,16 +51,6 @@ pub struct WheelFile<'a> { } impl<'a> WheelFile<'a> { - pub fn parse(path: &'a Path) -> anyhow::Result { - let file_name = path.file_name().and_then(OsStr::to_str).ok_or_else(|| { - anyhow!( - "Could not determine wheel filename from path: {path}", - path = path.display() - ) - })?; - Self::parse_file_name(file_name) - } - pub(crate) fn parse_file_name(file_name: &'a str) -> anyhow::Result { // See: https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-name-convention // {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl @@ -113,6 +104,46 @@ impl<'a> WheelFile<'a> { tags, }) } + + fn write_tag_component( + &self, + f: &mut Formatter<'_>, + extract_tag_component: impl Fn(&Tag<'a>) -> &'a str, + ) -> std::fmt::Result { + f.write_char('-')?; + for (idx, python) in self + .tags + .iter() + .map(extract_tag_component) + .collect::>() + .into_iter() + .enumerate() + { + if idx > 0 { + f.write_char('.')?; + } + f.write_str(python)?; + } + Ok(()) + } +} + +impl<'a> Display for WheelFile<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{project_name}-{version}", + project_name = self.raw_project_name, + version = self.raw_version + )?; + if let Some(build_tag) = self.build_tag { + write!(f, "-{build_tag}")?; + } + self.write_tag_component(f, |tag| tag.python)?; + self.write_tag_component(f, |tag| tag.abi)?; + self.write_tag_component(f, |tag| tag.platform)?; + Ok(()) + } } pub struct WheelMetadata<'a> { @@ -154,21 +185,6 @@ impl MetadataReader for WhlMetadataReader { } impl<'a> WheelMetadata<'a> { - pub fn try_from_path(path: &'a Path) -> anyhow::Result { - let file_name = path.file_name().and_then(OsStr::to_str).ok_or_else(|| { - anyhow!( - "Could not determine wheel filename from path: {path}", - path = path.display() - ) - })?; - let wheel_file = WheelFile::parse_file_name(file_name)?; - if path.is_dir() { - Self::parse(wheel_file, DirMetadataReader(path)) - } else { - Self::parse(wheel_file, WhlMetadataReader::new(path)?) - } - } - pub fn parse( wheel_file: WheelFile<'a>, mut metadata_reader: impl MetadataReader, @@ -202,7 +218,7 @@ impl<'a> WheelMetadata<'a> { #[cfg(test)] mod tests { - + use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; @@ -214,7 +230,7 @@ mod tests { use testing::{tmp_dir, venv_python_exe}; use zip::ZipArchive; - use crate::wheel::{Tag, WheelFile, WheelMetadata}; + use crate::wheel::{DirMetadataReader, Tag, WheelFile, WheelMetadata, WhlMetadataReader}; #[test] fn test_parse_wheel_file_name_simple() { @@ -322,7 +338,13 @@ mod tests { } fn assert_requests_2_32_5_whl(wheel: &Path) { - let wheel = WheelMetadata::try_from_path(wheel).unwrap(); + let file_name = wheel.file_name().and_then(OsStr::to_str).unwrap(); + let wheel_file = WheelFile::parse_file_name(file_name).unwrap(); + let wheel = if wheel.is_dir() { + WheelMetadata::parse(wheel_file, DirMetadataReader(wheel)).unwrap() + } else { + WheelMetadata::parse(wheel_file, WhlMetadataReader::new(wheel).unwrap()).unwrap() + }; assert_eq!("requests", wheel.wheel_file.raw_project_name); assert_eq!("2.32.5", wheel.wheel_file.raw_version); assert_eq!( diff --git a/crates/pexrs/Cargo.toml b/crates/pexrs/Cargo.toml index 0063745..eba9d3b 100644 --- a/crates/pexrs/Cargo.toml +++ b/crates/pexrs/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] anyhow = { workspace = true } cache = { path = "../cache" } +fs-err = { workspace = true } interpreter = { path = "../interpreter" } itertools = { workspace = true } log = { workspace = true } diff --git a/crates/pexrs/src/lib.rs b/crates/pexrs/src/lib.rs index 6a63a5c..7a602c9 100644 --- a/crates/pexrs/src/lib.rs +++ b/crates/pexrs/src/lib.rs @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; +use std::env; use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; use std::process::Command; -use std::{env, fs}; use anyhow::anyhow; use cache::{CacheDir, HashOptions, Key, atomic_dir}; @@ -15,7 +15,7 @@ use log::{debug, warn}; use logging_timer::time; use pex::{Pex, PexPath}; use regex::bytes::Regex; -use venv::{Virtualenv, populate, populate_wheels}; +use venv::{Virtualenv, populate, populate_user_code_and_wheels}; #[time("debug", "{}")] pub fn boot( @@ -43,6 +43,7 @@ fn prepare_boot( let venv = prepare_venv( python, pex.as_ref(), + #[cfg(unix)] env::var_os("_PEXRC_SH_BOOT_SEED_DIR").map(PathBuf::from), )?; let mut command = Command::new(&venv.interpreter.path); @@ -76,18 +77,23 @@ fn exec(command: &mut Command) -> anyhow::Result { #[time("debug", "{}")] pub fn mount(python: impl AsRef, pex: impl AsRef) -> anyhow::Result { - prepare_venv(python, pex.as_ref(), None).map(|venv| venv.site_packages_path()) + prepare_venv( + python, + pex.as_ref(), + #[cfg(unix)] + None, + ) + .map(|venv| venv.site_packages_path()) } #[time("debug", "{}")] fn prepare_venv<'a>( python: impl AsRef, pex: &'a Path, - sh_boot_seed_dir: Option, + #[cfg(unix)] sh_boot_seed_dir: Option, ) -> anyhow::Result> { let pex = Pex::load(pex)?; - let pex_info = pex.info(); - let pex_path = PexPath::from_pex_info(pex_info, true); + let pex_path = PexPath::from_pex_info(&pex.info, true); let additional_pexes = pex_path.load_pexes()?; let search_path = SearchPath::from_env()?; let venv_dir = venv_dir(Some(python.as_ref()), &pex, &search_path, &additional_pexes)?; @@ -98,11 +104,11 @@ fn prepare_venv<'a>( interpreter, Cow::Borrowed(work_dir), &mut resources, - pex_info.venv_system_site_packages, + pex.info.venv_system_site_packages, )?; populate(&venv, &venv_dir, &pex, &selected_wheels, &mut resources)?; for (additional_pex, selected_wheels) in additional_pexes { - populate_wheels(&venv, additional_pex, &selected_wheels, false)?; + populate_user_code_and_wheels(&venv, additional_pex, &selected_wheels, false)?; } Ok(venv.interpreter) })? { @@ -111,7 +117,7 @@ fn prepare_venv<'a>( venv_interpreter.store()?; #[cfg(unix)] if let Some(sh_boot_seed_dir) = sh_boot_seed_dir { - fs::create_dir_all(&sh_boot_seed_dir)?; + fs_err::create_dir_all(&sh_boot_seed_dir)?; platform::unix::symlink( venv_dir.join("pex"), sh_boot_seed_dir.join(venv_interpreter.most_specific_exe_name()), @@ -134,17 +140,16 @@ pub fn venv_dir( search_path: &SearchPath, additional_pexes: &[Pex], ) -> anyhow::Result { - let pex_info = pex.info(); let mut key = Key::default(); // The primary PEX hash covers its user code contents, distributions and ICs. - key.property("pex_hash", &pex_info.pex_hash); + key.property("pex_hash", &pex.info.pex_hash); // We hash just the distributions of additional PEXes since those are the only items used from // PEX_PATH adjoined PEX files; i.e.: neither the entry_point nor any other PEX file data or // metadata is used. for additional_pex in additional_pexes { - key.object("additional_pex", additional_pex.info().distributions.iter()); + key.object("additional_pex", additional_pex.info.distributions.iter()); } let mut imprecise_pex_python: Option<&OsStr> = None; @@ -152,7 +157,7 @@ pub fn venv_dir( // If there are no restrictions on interpreter, whatever we derive from the ambient python is // our opaque choice, which we can keep. - if pex_info.interpreter_constraints.is_empty() + if pex.info.interpreter_constraints.is_empty() && search_path.is_empty() && let Some(python_exe) = ambient_python { @@ -166,7 +171,7 @@ pub fn venv_dir( if let Some(python) = search_path.pex_python() { let value = python.as_encoded_bytes(); key.property("PEX_PYTHON", value); - if pex_info.emit_warnings + if pex.info.emit_warnings && !Regex::new(r"^(?:[Pp]ython|pypy)\d+\.\d+[^\d]?(?:\.exe)$")?.is_match(value) { imprecise_pex_python = Some(python); @@ -177,7 +182,7 @@ pub fn venv_dir( "PEX_PYTHON_PATH", path.iter().map(|path| path.as_os_str().as_encoded_bytes()), ); - if pex_info.emit_warnings { + if pex.info.emit_warnings { imprecise_pex_python_path = Some(env::join_paths(path)?); } } @@ -200,7 +205,7 @@ pub fn venv_dir( with `--no-emit-warnings` or re-run the PEX with PEX_EMIT_WARNINGS=False.\n\ ", pex_python = pex_python.display(), - pex_file = pex.path().display(), + pex_file = pex.path.display(), venv_dir = venv_dir.display() ) } @@ -220,7 +225,7 @@ pub fn venv_dir( with PEX_EMIT_WARNINGS=False.\n\ ", ppp = pex_python_path.display(), - pex_file = pex.path().display(), + pex_file = pex.path.display(), venv_dir = venv_dir.display() ) } diff --git a/crates/platform/src/lib.rs b/crates/platform/src/lib.rs index a7c52a4..b089a60 100644 --- a/crates/platform/src/lib.rs +++ b/crates/platform/src/lib.rs @@ -7,6 +7,7 @@ pub mod unix; #[cfg(windows)] mod windows; +use std::ffi::OsStr; use std::path::Path; use anyhow::anyhow; @@ -25,6 +26,15 @@ pub fn path_as_str(path: &Path) -> anyhow::Result<&str> { }) } +pub fn os_str_as_str(text: &OsStr) -> anyhow::Result<&str> { + text.to_str().ok_or_else(|| { + anyhow!( + "Failed to convert non-UTF8 text to str: {text}", + text = text.display() + ) + }) +} + pub fn link_or_copy(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { fs::hard_link(&src, &dst) .or_else(|_| fs::copy(src, dst).map(|_| ())) diff --git a/crates/platform/src/windows.rs b/crates/platform/src/windows.rs index fa5ba4d..fcf7a77 100644 --- a/crates/platform/src/windows.rs +++ b/crates/platform/src/windows.rs @@ -4,7 +4,6 @@ use std::fs::File; use std::path::Path; -use fs_err as fs; use is_executable::IsExecutable; pub fn symlink_or_link_or_copy( diff --git a/crates/resources/src/embedded.rs b/crates/resources/src/embedded.rs deleted file mode 100644 index 92f27d0..0000000 --- a/crates/resources/src/embedded.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2026 Pex project contributors. -// SPDX-License-Identifier: Apache-2.0 - -use std::borrow::Cow; - -use crate::{ResourcePath, Resources}; - -pub struct EmbeddedResources(()); - -impl Resources<'static> for EmbeddedResources { - fn read(&mut self, path: ResourcePath) -> anyhow::Result> { - Ok(Cow::Borrowed(match path { - ResourcePath::InterpreterIdentificationScript => include_str!("interpreter.py"), - ResourcePath::VendoredVirtualenvScript => include_str!(env!("VIRTUALENV_PY")), - ResourcePath::VenvPexScript => include_str!("venv-pex.py"), - ResourcePath::VenvPexReplScript => include_str!("venv-pex-repl.py"), - })) - } -} - -pub const RESOURCES: EmbeddedResources = EmbeddedResources(()); diff --git a/crates/resources/src/lib.rs b/crates/resources/src/lib.rs deleted file mode 100644 index ab26520..0000000 --- a/crates/resources/src/lib.rs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2026 Pex project contributors. -// SPDX-License-Identifier: Apache-2.0 - -use std::borrow::Cow; -use std::io::{Seek, Write}; - -use strum::IntoEnumIterator; -use strum_macros::EnumIter; -use zip::ZipWriter; -use zip::write::{FileOptionExtension, FileOptions, SimpleFileOptions}; - -#[cfg(feature = "embedded")] -pub mod embedded; - -#[derive(Copy, Clone, EnumIter)] -pub enum ResourcePath { - InterpreterIdentificationScript, - VendoredVirtualenvScript, - VenvPexScript, - VenvPexReplScript, -} - -impl ResourcePath { - pub fn script_name(&self) -> &'static str { - match self { - ResourcePath::InterpreterIdentificationScript => "interpreter.py", - ResourcePath::VendoredVirtualenvScript => "virtualenv.py", - ResourcePath::VenvPexScript => "venv-pex.py", - ResourcePath::VenvPexReplScript => "venv-pex-repl.py", - } - } -} - -pub trait Resources<'a> { - fn read(&mut self, path: ResourcePath) -> anyhow::Result>; - - fn inject_scripts( - &mut self, - zip: &mut ZipWriter, - file_options: FileOptions<'a, T>, - ) -> anyhow::Result<()> { - let directory_options = SimpleFileOptions::default(); - zip.add_directory("__pex__/.scripts", directory_options)?; - for resource_path in ResourcePath::iter() { - let text = self.read(resource_path)?; - zip.start_file( - format!( - "__pex__/.scripts/{script}", - script = resource_path.script_name() - ), - file_options, - )?; - zip.write_all(text.as_bytes())?; - } - Ok(()) - } -} - -macro_rules! generate_script_type { - ( $resource_path:ident ) => { - pub struct $resource_path<'a>(Cow<'a, str>); - - impl<'a> $resource_path<'a> { - pub fn read(resources: &mut impl Resources<'a>) -> anyhow::Result<$resource_path<'a>> { - let text = resources.read(ResourcePath::$resource_path)?; - Ok($resource_path(text)) - } - - pub fn contents(&self) -> &str { - self.0.as_ref() - } - } - }; -} - -generate_script_type!(InterpreterIdentificationScript); -generate_script_type!(VendoredVirtualenvScript); -generate_script_type!(VenvPexScript); -generate_script_type!(VenvPexReplScript); diff --git a/crates/resources/Cargo.toml b/crates/scripts/Cargo.toml similarity index 69% rename from crates/resources/Cargo.toml rename to crates/scripts/Cargo.toml index 226be79..bc35d7b 100644 --- a/crates/resources/Cargo.toml +++ b/crates/scripts/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "resources" +name = "scripts" edition = "2024" [features] @@ -7,11 +7,13 @@ embedded = [] [dependencies] anyhow = { workspace = true} +fs-err = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } zip = { workspace = true } [build-dependencies] anyhow = { workspace = true } +build-system = { path = "../build-system" } env_logger = { workspace = true } -pexrc-build-system = { path = "../pexrc-build-system" } \ No newline at end of file +fs-err = { workspace = true } \ No newline at end of file diff --git a/crates/resources/build.rs b/crates/scripts/build.rs similarity index 94% rename from crates/resources/build.rs rename to crates/scripts/build.rs index 9a97093..352dc60 100644 --- a/crates/resources/build.rs +++ b/crates/scripts/build.rs @@ -1,10 +1,11 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 +use std::env; use std::path::PathBuf; -use std::{env, fs}; -use pexrc_build_system::{InstallDirs, download_virtualenv}; +use build_system::{InstallDirs, download_virtualenv}; +use fs_err as fs; fn main() -> anyhow::Result<()> { env_logger::init(); diff --git a/crates/resources/src/interpreter.py b/crates/scripts/src/interpreter.py similarity index 99% rename from crates/resources/src/interpreter.py rename to crates/scripts/src/interpreter.py index b84c592..8c9a300 100644 --- a/crates/resources/src/interpreter.py +++ b/crates/scripts/src/interpreter.py @@ -649,9 +649,10 @@ def cpython_abis(py_version): try: # N.B.: There is no importlib.machinery prior to ~3.3. - from importlib.machinery import EXTENSION_SUFFIXES # type: ignore + from importlib.machinery import EXTENSION_SUFFIXES # type: ignore[import-not-found] except ImportError: - import imp + # N.B.: There is no imp from 3.12 on. + import imp # type: ignore[import-not-found] EXTENSION_SUFFIXES = [x[0] for x in imp.get_suffixes()] del imp diff --git a/crates/scripts/src/lib.rs b/crates/scripts/src/lib.rs new file mode 100644 index 0000000..0ba867a --- /dev/null +++ b/crates/scripts/src/lib.rs @@ -0,0 +1,125 @@ +// Copyright 2026 Pex project contributors. +// SPDX-License-Identifier: Apache-2.0 + +use std::borrow::Cow; +use std::io; +use std::io::{Seek, Write}; +use std::iter::Iterator; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; + +use fs_err as fs; +use fs_err::File; +use strum::IntoEnumIterator; +use strum_macros::EnumIter; +use zip::write::{FileOptionExtension, FileOptions, SimpleFileOptions}; +use zip::{ZipArchive, ZipWriter}; + +#[derive(Copy, Clone, EnumIter)] +pub enum Script { + IdentifyInterpreter, + VendoredVirtualenv, + VenvPex, + VenvPexRepl, +} + +impl Script { + pub const fn file_name(&self) -> &'static str { + match self { + Script::IdentifyInterpreter => "interpreter.py", + Script::VendoredVirtualenv => "virtualenv.py", + Script::VenvPex => "venv-pex.py", + Script::VenvPexRepl => "venv-pex-repl.py", + } + } +} + +pub enum Scripts { + #[cfg(feature = "embedded")] + Embedded, + Loose(PathBuf), + Zipped(ZipArchive), +} + +const ZIP_REL_PATH: &str = "__pex__/.scripts"; +static HOST_REL_PATH: LazyLock = LazyLock::new(|| ZIP_REL_PATH.split("/").collect()); + +impl Scripts { + pub fn read(&mut self, script: Script) -> anyhow::Result> { + match self { + #[cfg(feature = "embedded")] + Scripts::Embedded => Ok(Cow::Borrowed(match script { + Script::IdentifyInterpreter => include_str!("interpreter.py"), + Script::VendoredVirtualenv => include_str!(env!("VIRTUALENV_PY")), + Script::VenvPex => include_str!("venv-pex.py"), + Script::VenvPexRepl => include_str!("venv-pex-repl.py"), + })), + Scripts::Loose(base_dir) => { + let resource_path = base_dir + .join(HOST_REL_PATH.as_path()) + .join(script.file_name()); + Ok(Cow::Owned(fs::read_to_string(resource_path)?)) + } + Scripts::Zipped(zip) => { + let resource_path = + format!("{ZIP_REL_PATH}/{file_name}", file_name = script.file_name()); + let entry = zip.by_name(&resource_path)?; + Ok(Cow::Owned(io::read_to_string(entry)?)) + } + } + } + + pub fn inject_scripts<'a, T: FileOptionExtension + Copy>( + &mut self, + zip: &'a mut ZipWriter, + file_options: FileOptions<'a, T>, + ) -> anyhow::Result<()> { + let directory_options = SimpleFileOptions::default(); + zip.add_directory(ZIP_REL_PATH, directory_options)?; + for resource_path in Script::iter() { + let text = self.read(resource_path)?; + zip.start_file( + format!( + "{ZIP_REL_PATH}/{script}", + script = resource_path.file_name() + ), + file_options, + )?; + zip.write_all(text.as_bytes())?; + } + Ok(()) + } + + pub fn write_scripts(&mut self, dest_dir: &Path) -> anyhow::Result<()> { + let scripts_dir = dest_dir.join(HOST_REL_PATH.as_path()); + fs::create_dir_all(&scripts_dir)?; + for resource_path in Script::iter() { + let text = self.read(resource_path)?; + let mut file = File::create_new(scripts_dir.join(resource_path.file_name()))?; + file.write_all(text.as_bytes())?; + } + Ok(()) + } +} + +macro_rules! generate_script_type { + ( $script_type:ident ) => { + pub struct $script_type<'a>(Cow<'a, str>); + + impl<'a> $script_type<'a> { + pub fn read(scripts: &mut Scripts) -> anyhow::Result<$script_type<'a>> { + let text = scripts.read(Script::$script_type)?; + Ok($script_type(text)) + } + + pub fn contents(&self) -> &str { + self.0.as_ref() + } + } + }; +} + +generate_script_type!(IdentifyInterpreter); +generate_script_type!(VendoredVirtualenv); +generate_script_type!(VenvPex); +generate_script_type!(VenvPexRepl); diff --git a/crates/resources/src/venv-pex-repl.py b/crates/scripts/src/venv-pex-repl.py similarity index 100% rename from crates/resources/src/venv-pex-repl.py rename to crates/scripts/src/venv-pex-repl.py diff --git a/crates/resources/src/venv-pex.py b/crates/scripts/src/venv-pex.py similarity index 97% rename from crates/resources/src/venv-pex.py rename to crates/scripts/src/venv-pex.py index 7f5c0d9..e237e00 100644 --- a/crates/resources/src/venv-pex.py +++ b/crates/scripts/src/venv-pex.py @@ -314,8 +314,8 @@ def __init__( self, seed=(), # type: Union[Mapping[str, Any], Iterable[Tuple[str, Any]]] safe=False, # type: bool - **kwargs, # type: Any - ): + **kwargs # type: Any + ): # fmt: skip # type: (...) -> None self.__dict__.update(seed) self.__dict__.update(kwargs) @@ -414,7 +414,15 @@ def _value(self, key): with open(file_path) as fp: content = fp.read() - ast = compile(content, filename, "exec", flags=0, dont_inherit=1) + # N.B.: MyPy doesn't track the union type of content above correctly across all versions + # of Python we support. + ast = compile( + content, + filename, + "exec", + flags=0, + dont_inherit=1, + ) # type: ignore[call-overload] globals_map = globals().copy() globals_map["__name__"] = "__main__" globals_map["__file__"] = filename diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 3423e8c..9839f99 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -7,7 +7,7 @@ anyhow = { workspace = true } cache = { path = "../cache" } ctor = { workspace = true } fs-err = { workspace = true } -resources = { path = "../resources", features = ["embedded"] } rstest = { workspace = true } +scripts = { path = "../scripts", features = ["embedded"] } target-lexicon = { workspace = true } tempfile = { workspace = true } diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index a816c5d..6d4a2b3 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -9,8 +9,8 @@ use std::sync::{LazyLock, Mutex}; use anyhow::anyhow; use ctor::dtor; use fs_err as fs; -use resources::{InterpreterIdentificationScript, Resources, embedded}; use rstest::fixture; +use scripts::{IdentifyInterpreter, Scripts}; use target_lexicon::{HOST, OperatingSystem}; static TMP_DIRS: Mutex> = Mutex::new(Vec::new()); @@ -119,13 +119,13 @@ pub fn venv_python_exe(python_exe: &Path) -> PathBuf { } #[fixture] -pub fn embedded_resources() -> impl Resources<'static> { - embedded::RESOURCES +pub fn embedded_scripts() -> Scripts { + Scripts::Embedded } #[fixture] pub fn interpreter_identification_script( - mut embedded_resources: impl Resources<'static>, -) -> InterpreterIdentificationScript<'static> { - InterpreterIdentificationScript::read(&mut embedded_resources).unwrap() + mut embedded_scripts: Scripts, +) -> IdentifyInterpreter<'static> { + IdentifyInterpreter::read(&mut embedded_scripts).unwrap() } diff --git a/crates/venv/Cargo.toml b/crates/venv/Cargo.toml index 568e2a8..ad3d67a 100644 --- a/crates/venv/Cargo.toml +++ b/crates/venv/Cargo.toml @@ -15,16 +15,17 @@ log = { workspace = true } logging_timer = { workspace = true } pex = { path = "../pex" } platform = { path = "../platform" } -resources = { path = "../resources" } rayon = { workspace = true } +scripts = { path = "../scripts" } target-lexicon = { workspace = true } tempfile = { workspace = true } +walkdir = { workspace = true } zip = { workspace = true } [dev-dependencies] interpreter = { path = "../interpreter" } -resources = { path = "../resources" } rstest = { workspace = true} +scripts = { path = "../scripts" } testing = { path = "../testing" } [build-dependencies] diff --git a/crates/venv/src/lib.rs b/crates/venv/src/lib.rs index f26c7c6..156918c 100644 --- a/crates/venv/src/lib.rs +++ b/crates/venv/src/lib.rs @@ -4,5 +4,5 @@ pub mod venv_pex; pub mod virtualenv; -pub use venv_pex::{populate, populate_wheels}; +pub use venv_pex::{populate, populate_user_code_and_wheels}; pub use virtualenv::Virtualenv; diff --git a/crates/venv/src/venv_pex.rs b/crates/venv/src/venv_pex.rs index dd6049d..d150c17 100644 --- a/crates/venv/src/venv_pex.rs +++ b/crates/venv/src/venv_pex.rs @@ -1,76 +1,166 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashSet; use std::fmt::{Display, Formatter}; +use std::io; use std::io::Write; -use std::path::Path; -use std::{fs, io}; +use std::path::{Path, PathBuf}; use anyhow::anyhow; -use fs_err::File; +use fs_err as fs; +use fs_err::{DirEntry, File}; use indexmap::{IndexMap, IndexSet}; use log::warn; use logging_timer::time; -use pex::{BinPath, InheritPath, Pex, PexInfo}; +use pex::{BinPath, InheritPath, Layout, Pex, PexInfo}; use platform::{mark_executable, path_as_bytes, path_as_str, symlink_or_link_or_copy}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use resources::{Resources, VenvPexReplScript, VenvPexScript}; +use scripts::{Scripts, VenvPex, VenvPexRepl}; use zip::ZipArchive; use crate::virtualenv::Virtualenv; -#[time("debug", "{}")] -pub fn populate_wheels<'a>( +fn populate_wheel(wheel: &Path, site_packages_path: &Path) -> anyhow::Result<()> { + let whl_zip = ZipArchive::new(File::open(wheel)?)?; + let metadata = whl_zip.metadata(); + (0..whl_zip.len()).into_par_iter().try_for_each(|index| { + let zip_fp = File::open(wheel)?; + let mut zip = unsafe { ZipArchive::unsafe_new_with_metadata(zip_fp, metadata.clone()) }; + extract_idx(site_packages_path, index, &mut zip)?; + Ok(()) + }) +} + +fn populate_from_packed_pex<'a>( venv: &Virtualenv, - pex: &'a Pex<'a>, - selected_wheels: &IndexSet<&'a str>, + packed_pex: &'a Pex<'a>, + selected_wheels: &IndexSet<&str>, populate_pex_info: bool, -) -> anyhow::Result<&'a PexInfo> { +) -> anyhow::Result<()> { let site_packages_path = venv.site_packages_path(); - match pex { - Pex::Loose(_) => todo!("XXX: Implement loose PEX venv population."), - Pex::Packed(_) => todo!("XXX: Implement packed PEX venv population."), - Pex::ZipApp(zip_app_pex) => { - let mut pex_zip = ZipArchive::new(File::open(zip_app_pex.0)?)?; - let metadata = pex_zip.metadata(); - let extract_indexes = pex_zip - .file_names() - .enumerate() - .filter_map(|(idx, name)| { - // TODO: XXX: Deal with .layout and .prefix/ in wheel chroots. - if [".bootstrap/", "__pex__/"] - .iter() - .any(|exclude_dir| name.starts_with(exclude_dir)) - || ["PEX-INFO", "__main__.py", ".deps/"].contains(&name) - || name.starts_with(".deps/") - && name[6..] - .split("/") - .next() - .map(|whl_name| !selected_wheels.contains(whl_name)) - .unwrap_or(true) - { - None - } else { - Some(idx) - } - }) - .collect::>(); - extract_indexes - .into_par_iter() - .try_for_each(|index| -> anyhow::Result<()> { - let zip_fp = File::open(zip_app_pex.0)?; - let mut zip = - unsafe { ZipArchive::unsafe_new_with_metadata(zip_fp, metadata.clone()) }; - extract_idx(&site_packages_path, index, &mut zip)?; - Ok(()) - })?; - if populate_pex_info { - let mut pex_info_src_fp = pex_zip.by_name("PEX-INFO")?; - let mut pex_info_dst_fp = File::create_new(venv.prefix().join("PEX-INFO"))?; - io::copy(&mut pex_info_src_fp, &mut pex_info_dst_fp)?; + + let deps_dir = packed_pex.path.join(".deps"); + if deps_dir.is_dir() { + let wheels = fs::read_dir(deps_dir)? + .filter(|result| match result { + Ok(entry) => platform::os_str_as_str(&entry.file_name()) + .map(|whl| selected_wheels.contains(whl)) + .unwrap_or_default(), + Err(_) => true, + }) + .collect::, _>>()?; + wheels + .into_par_iter() + .try_for_each(|wheel| populate_wheel(&wheel.path(), &site_packages_path))?; + } + + let excludes: HashSet = [ + ".bootstrap", + ".deps", + "__main__.py", + "__pex__", + "__pycache__", + "pex", + "pex-repl", + "PEX-INFO", + "PEX-LAYOUT", + ] + .into_iter() + .map(|rel_path| packed_pex.path.join(rel_path)) + .collect(); + let _pex_info_exclude = if populate_pex_info { + None + } else { + Some(packed_pex.path.join("PEX-INFO")) + }; + let user_code = walkdir::WalkDir::new(packed_pex.path) + .min_depth(1) + .into_iter() + .filter_entry(|entry| !excludes.contains(entry.path())) + .collect::, _>>()?; + user_code.into_par_iter().try_for_each(|entry| { + if entry.file_type().is_dir() { + fs::create_dir_all(entry.path()) + } else { + if let Some(parent) = entry.path().parent() { + fs::create_dir_all(parent)?; } - Ok(&zip_app_pex.1) + let dst = site_packages_path.join(entry.path().strip_prefix(packed_pex.path).expect( + "Walked packed PEX paths should be child paths of the packed PEX root dir.", + )); + fs::copy(entry.path(), dst).map(|_| ()) } + })?; + + if populate_pex_info { + fs::copy( + packed_pex.path.join("PEX-INFO"), + venv.prefix().join("PEX-INFO"), + )?; + } + + Ok(()) +} + +fn populate_from_zip_app<'a>( + venv: &Virtualenv, + zip_app_pex: &'a Pex<'a>, + selected_wheels: &IndexSet<&'a str>, + populate_pex_info: bool, +) -> anyhow::Result<()> { + let mut pex_zip = ZipArchive::new(File::open(zip_app_pex.path)?)?; + let metadata = pex_zip.metadata(); + let extract_indexes = pex_zip + .file_names() + .enumerate() + .filter_map(|(idx, name)| { + // TODO: XXX: Deal with .layout and .prefix/ in wheel chroots. + if [".bootstrap/", "__pex__/"] + .iter() + .any(|exclude_dir| name.starts_with(exclude_dir)) + || ["PEX-INFO", "__main__.py", ".deps/"].contains(&name) + || name.starts_with(".deps/") + && name[6..] + .split("/") + .next() + .map(|whl_name| !selected_wheels.contains(whl_name)) + .unwrap_or(true) + { + None + } else { + Some(idx) + } + }) + .collect::>(); + let site_packages_path = venv.site_packages_path(); + extract_indexes + .into_par_iter() + .try_for_each(|index| -> anyhow::Result<()> { + let zip_fp = File::open(zip_app_pex.path)?; + let mut zip = unsafe { ZipArchive::unsafe_new_with_metadata(zip_fp, metadata.clone()) }; + extract_idx(&site_packages_path, index, &mut zip)?; + Ok(()) + })?; + if populate_pex_info { + let mut pex_info_src_fp = pex_zip.by_name("PEX-INFO")?; + let mut pex_info_dst_fp = File::create_new(venv.prefix().join("PEX-INFO"))?; + io::copy(&mut pex_info_src_fp, &mut pex_info_dst_fp)?; + } + Ok(()) +} + +#[time("debug", "{}")] +pub fn populate_user_code_and_wheels<'a>( + venv: &Virtualenv, + pex: &'a Pex<'a>, + selected_wheels: &IndexSet<&'a str>, + populate_pex_info: bool, +) -> anyhow::Result<()> { + match pex.layout { + Layout::Loose => todo!("Loose PEX venv population."), + Layout::Packed => populate_from_packed_pex(venv, pex, selected_wheels, populate_pex_info), + Layout::ZipApp => populate_from_zip_app(venv, pex, selected_wheels, populate_pex_info), } } @@ -80,36 +170,32 @@ pub fn populate<'a>( resting_venv_dir: &Path, pex: &'a Pex<'a>, selected_wheels: &IndexSet<&str>, - resources: &mut impl Resources<'a>, + scripts: &mut Scripts, ) -> anyhow::Result<()> { - let path = match &pex { - Pex::Loose(_) => todo!("XXX: Implement loose PEX venv population."), - Pex::Packed(_) => todo!("XXX: Implement packed PEX venv population."), - Pex::ZipApp(zip_app_pex) => zip_app_pex.0, - }; - let pex_info = populate_wheels(venv, pex, selected_wheels, true)?; + populate_user_code_and_wheels(venv, pex, selected_wheels, true)?; + let interpreter_relpath = venv .interpreter .path .strip_prefix(&venv.interpreter.prefix)?; let shebang_interpreter = resting_venv_dir.join(interpreter_relpath); - let shebang_arg = if (pex_info.venv && pex_info.venv_hermetic_scripts) - || (!pex_info.venv - && pex_info.inherit_path.unwrap_or(InheritPath::False) == InheritPath::False) + let shebang_arg = if (pex.info.venv && pex.info.venv_hermetic_scripts) + || (!pex.info.venv + && pex.info.inherit_path.unwrap_or(InheritPath::False) == InheritPath::False) { Some(venv.interpreter.hermetic_args()) } else { None }; - write_main(venv, &shebang_interpreter, shebang_arg, pex_info, resources)?; + write_main(venv, &shebang_interpreter, shebang_arg, &pex.info, scripts)?; write_repl( venv, &shebang_interpreter, shebang_arg, - path, - pex_info, + pex.path, + &pex.info, selected_wheels, - resources, + scripts, ) } @@ -209,17 +295,17 @@ impl<'a> Display for PythonListTupleStrStr<'a> { } } -fn write_main<'a>( +fn write_main( venv: &Virtualenv, shebang_interpreter: &Path, shebang_arg: Option<&str>, pex_info: &PexInfo, - resources: &mut impl Resources<'a>, + scripts: &mut Scripts, ) -> anyhow::Result<()> { let main_py = venv.prefix().join("__main__.py"); let mut main_py_fp = File::create_new(&main_py)?; write_shebang_bytes(&mut main_py_fp, shebang_interpreter, shebang_arg)?; - let venv_pex_script = VenvPexScript::read(resources)?; + let venv_pex_script = VenvPex::read(scripts)?; main_py_fp.write_all(venv_pex_script.contents().as_bytes())?; write!( @@ -266,18 +352,18 @@ if __name__ == "__main__": symlink_or_link_or_copy(&main_py, venv.prefix().join("pex"), true) } -fn write_repl<'a>( +fn write_repl( venv: &Virtualenv, shebang_interpreter: &Path, shebang_arg: Option<&str>, pex: &Path, pex_info: &PexInfo, selected_wheels: &IndexSet<&str>, - resources: &mut impl Resources<'a>, + scripts: &mut Scripts, ) -> anyhow::Result<()> { let mut pex_repl_py_fp = File::create_new(venv.prefix().join("pex-repl"))?; write_shebang_bytes(&mut pex_repl_py_fp, shebang_interpreter, shebang_arg)?; - let venv_pex_repl_script = VenvPexReplScript::read(resources)?; + let venv_pex_repl_script = VenvPexRepl::read(scripts)?; pex_repl_py_fp.write_all(venv_pex_repl_script.contents().as_bytes())?; let activation_summary = if selected_wheels.is_empty() { diff --git a/crates/venv/src/virtualenv.rs b/crates/venv/src/virtualenv.rs index 81d1109..fa5948d 100644 --- a/crates/venv/src/virtualenv.rs +++ b/crates/venv/src/virtualenv.rs @@ -2,18 +2,19 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; +use std::env; use std::env::consts::EXE_SUFFIX; use std::io::Write; use std::path::{MAIN_SEPARATOR_STR, Path, PathBuf}; use std::process::{Command, Stdio}; -use std::{env, fs}; use anyhow::{anyhow, bail}; use const_format::concatcp; +use fs_err as fs; use interpreter::Interpreter; use logging_timer::time; use platform::symlink_or_link_or_copy; -use resources::{InterpreterIdentificationScript, Resources, VendoredVirtualenvScript}; +use scripts::{IdentifyInterpreter, Scripts, VendoredVirtualenv}; use target_lexicon::{HOST, OperatingSystem}; const SCRIPTS_DIR: &str = env!("SCRIPTS_DIR"); @@ -38,8 +39,8 @@ impl<'a> Virtualenv<'a> { } #[time("debug", "Virtualenv.{}")] - pub fn load(path: Cow<'a, Path>, resources: &mut impl Resources<'a>) -> anyhow::Result { - let identification_script = InterpreterIdentificationScript::read(resources)?; + pub fn load(path: Cow<'a, Path>, scripts: &mut Scripts) -> anyhow::Result { + let identification_script = IdentifyInterpreter::read(scripts)?; let interpreter = Interpreter::load( path.as_ref().join(VENV_PYTHON_RELPATH), &identification_script, @@ -64,7 +65,7 @@ impl<'a> Virtualenv<'a> { pub fn create( interpreter: Interpreter, path: Cow<'a, Path>, - resources: &mut impl Resources<'a>, + scripts: &mut Scripts, include_system_site_packages: bool, ) -> anyhow::Result { let venv_interpreter = Self::host_interpreter(path.as_ref(), &interpreter); @@ -75,10 +76,10 @@ impl<'a> Virtualenv<'a> { interpreter, path.as_ref(), include_system_site_packages, - resources, + scripts, )? } else { - let virtualenv_script = VendoredVirtualenvScript::read(resources)?; + let virtualenv_script = VendoredVirtualenv::read(scripts)?; create_virtualenv_venv( &interpreter, path.as_ref(), @@ -107,10 +108,10 @@ fn create_pep_405_venv<'a>( interpreter: Interpreter, path: &Path, include_system_site_packages: bool, - resources: &mut impl Resources<'a>, + scripts: &mut Scripts, ) -> anyhow::Result> { // See: https://peps.python.org/pep-0405/ - let base_interpreter = interpreter.resolve_base_interpreter(resources)?; + let base_interpreter = interpreter.resolve_base_interpreter(scripts)?; let home = base_interpreter.realpath.parent().ok_or_else(|| { anyhow!( "Failed to calculate the home dir of venv base python {path}", @@ -142,7 +143,7 @@ fn create_pep_405_venv<'a>( fn create_virtualenv_venv<'a>( interpreter: &Interpreter, path: &Path, - virtualenv_script: VendoredVirtualenvScript<'a>, + virtualenv_script: VendoredVirtualenv<'a>, include_system_site_packages: bool, ) -> anyhow::Result> { let mut script = tempfile::Builder::new() @@ -211,20 +212,15 @@ mod tests { use std::path::PathBuf; use interpreter::Interpreter; - use resources::{InterpreterIdentificationScript, Resources}; use rstest::rstest; - use testing::{embedded_resources, python_exe, tmp_dir}; + use scripts::{IdentifyInterpreter, Scripts}; + use testing::{embedded_scripts, python_exe, tmp_dir}; use crate::virtualenv::{Path, Virtualenv}; #[rstest] - fn test_create( - python_exe: &Path, - tmp_dir: PathBuf, - mut embedded_resources: impl Resources<'static>, - ) { - let identification_script = - InterpreterIdentificationScript::read(&mut embedded_resources).unwrap(); + fn test_create(python_exe: &Path, tmp_dir: PathBuf, mut embedded_scripts: Scripts) { + let identification_script = IdentifyInterpreter::read(&mut embedded_scripts).unwrap(); let interpreter = Interpreter::load(python_exe, &identification_script).unwrap(); let expected_prefix = interpreter .base_prefix @@ -234,7 +230,7 @@ mod tests { let venv = Virtualenv::create( interpreter, Cow::Owned(tmp_dir), - &mut embedded_resources, + &mut embedded_scripts, false, ) .unwrap(); diff --git a/pexrc.rs b/pexrc.rs index aaec566..1d1d5df 100644 --- a/pexrc.rs +++ b/pexrc.rs @@ -34,8 +34,8 @@ enum Commands { #[arg(value_parser=clap::builder::PossibleValuesParser::new(CLIB_BY_TARGET.keys()))] targets: Vec, - #[arg(value_name = "FILE")] - pex: PathBuf, + #[arg(value_name = "FILE", required = true)] + pexes: Vec, }, /// Provide information about the supported target runtimes. Info, @@ -50,7 +50,7 @@ fn main() -> anyhow::Result<()> { match cli.command { Commands::Inject { - pex, + pexes, compression_level, targets, } => { @@ -69,7 +69,10 @@ fn main() -> anyhow::Result<()> { } else { None }; - inject::inject(&pex, compression_level, clibs) + for pex in pexes { + inject::inject(&pex, compression_level, clibs.as_ref())?; + } + Ok(()) } Commands::Info => info::display(), } diff --git a/pyproject.toml b/pyproject.toml index 6ac3d9c..4bef319 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,11 @@ line-length = 100 extend-select = ["I"] [tool.dev-cmd.commands] -fmt = ["ruff", "format"] -check-fmt = ["ruff", "format", "--diff"] +fmt = ["ruff", "format", "crates/scripts", "python"] +check-fmt = ["ruff", "format", "--diff", "crates/scripts", "python"] -lint = ["ruff", "check", "--fix"] -check-lint = ["ruff", "check"] +lint = ["ruff", "check", "--fix", "crates/scripts", "python"] +check-lint = ["ruff", "check", "crates/scripts", "python"] [tool.dev-cmd.commands.type-check-pexrc.factors] py = "The Python version to type check in . form; i.e.: 3.13." @@ -38,6 +38,7 @@ args = [ "mypy", "--python-version", "{-py:{markers.python_version}}", "--cache-dir", ".mypy_cache_{markers.python_version}", + "crates/scripts", "python", ] @@ -50,7 +51,7 @@ args = ["scripts/generate-release-hashes.py"] accepts-extra-args = true [tool.dev-cmd.commands.test] -args = ["python/scripts/run-tests.py"] +args = ["python/testing/bin/run-tests.py"] accepts-extra-args = true [tool.dev-cmd.tasks.checks] diff --git a/python/testing/__init__.py b/python/testing/__init__.py index b30ebe9..9c5c42f 100644 --- a/python/testing/__init__.py +++ b/python/testing/__init__.py @@ -4,8 +4,11 @@ from __future__ import absolute_import import os +import platform import subprocess +IS_WINDOWS = platform.system().lower() == "windows" + def pexrc(): # type: () -> str @@ -17,5 +20,7 @@ def pexrc_inject(pex): subprocess.check_call(args=[pexrc(), "inject", pex]) injected_pex = pex + "rc" if pex.endswith(".pex") else pex + ".pexrc" - assert os.path.isfile(injected_pex) + assert (os.path.isfile(pex) and os.path.isfile(injected_pex)) or ( + os.path.isdir(pex) and os.path.isdir(injected_pex) + ) return injected_pex diff --git a/python/scripts/run-tests.py b/python/testing/bin/run-tests.py similarity index 100% rename from python/scripts/run-tests.py rename to python/testing/bin/run-tests.py diff --git a/python/testing/compare.py b/python/testing/compare.py index 4c9c421..21f607a 100644 --- a/python/testing/compare.py +++ b/python/testing/compare.py @@ -103,6 +103,7 @@ def compare( env=None, # type: Optional[Mapping[str, str]] test_result=None, # type: Optional[Callable[[ProcessResult, bool], None]] compare_results=None, # type: Optional[Callable[[ProcessResult, ProcessResult], None]] + injected_pex=None, # type: Optional[str] ): # type: (...) -> str @@ -113,8 +114,8 @@ def compare( file=sys.stderr, ) - injected_pex = pexrc_inject(pex) - injected_result = execute_pex(injected_pex, python_args, args, **(env or {})) + injected = injected_pex or pexrc_inject(pex) + injected_result = execute_pex(injected, python_args, args, **(env or {})) _test_result(injected_result, False, test_result=test_result) print( "Injected PEXRC run took {elapsed:.5}ms".format(elapsed=injected_result.elapsed * 1000), @@ -134,4 +135,4 @@ def compare( file=sys.stderr, ) _compare_results(traditional_result, injected_result, compare_results=compare_results) - return injected_pex + return injected diff --git a/python/tests/test_boot.py b/python/tests/test_boot.py index bbc23db..58dd2be 100644 --- a/python/tests/test_boot.py +++ b/python/tests/test_boot.py @@ -6,6 +6,7 @@ import os.path import subprocess +from testing import IS_WINDOWS, pexrc_inject from testing.compare import compare TYPE_CHECKING = False @@ -77,10 +78,66 @@ def test_sh_boot(tmpdir): expected_shebang = read_shebang(pex) assert expected_shebang == "#!/bin/sh\n" + pexrc_env = dict(PEXRC_ROOT=os.path.join(str(tmpdir), "pexrc-root")) injected_pex = compare( pex, args=["Moo!"], - env=dict(PEXRC_ROOT=os.path.join(str(tmpdir), "pexrc-root")), + env=pexrc_env, test_result=assert_result, ) assert expected_shebang == read_shebang(injected_pex) + + # N.B.: The above uses compare which executes python against the PEX, which just proves the + # `--sh-boot` shebang does not interfere with that. As long as we're not on Windows, we can run + # the `--sh-boot` shebang directly. + if not IS_WINDOWS: + assert b"| Moo! |" in subprocess.check_output(args=[pex, "Moo!"]) + assert b"| Moo! |" in subprocess.check_output(args=[injected_pex, "Moo!"], env=pexrc_env) + + +def test_packed(tmpdir): + # type: (Any) -> None + + pex = create_cowsay_pex(tmpdir, "--layout", "packed") + assert os.path.isdir(pex) + + injected_pex = compare( + pex=pex, + args=["Moo!"], + env=dict(PEXRC_ROOT=os.path.join(str(tmpdir), "pexrc-root")), + test_result=assert_result, + ) + assert os.path.isdir(injected_pex) + + +def test_packed_sh_boot(tmpdir): + # type: (Any) -> None + + pex = create_cowsay_pex(tmpdir, "--layout", "packed", "--sh-boot") + assert os.path.isdir(pex) + pex_script = os.path.join(pex, "pex") + # N.B.: Pex incorrectly uses the host line ending here, which we tolerate for the purposes of + # this test. A /bin/sh script, though, should always use \n line endings. + assert "#!/bin/sh{eol}".format(eol=os.linesep) == read_shebang(pex_script) + + injected_pex = pexrc_inject(pex) + assert os.path.isdir(injected_pex) + injected_pex_script = os.path.join(injected_pex, "pex") + assert "#!/bin/sh\n" == read_shebang(injected_pex_script) + + if not IS_WINDOWS: + pexrc_env = dict(PEXRC_ROOT=os.path.join(str(tmpdir), "pexrc-root")) + + # N.B.: The `--layout packed --sh-boot` PEXes Pex builds are broken on Windows; so we just + # run the comparison on unix. + compare( + pex, args=["Moo!"], env=pexrc_env, test_result=assert_result, injected_pex=injected_pex + ) + + # N.B.: The above uses `compare` which executes python against the PEX, which just proves + # the `--sh-boot` shebang does not interfere with that. As long as we're not on Windows, we + # can run the `--sh-boot` shebang directly. + assert b"| Moo! |" in subprocess.check_output(args=[pex_script, "Moo!"]) + assert b"| Moo! |" in subprocess.check_output( + args=[injected_pex_script, "Moo!"], env=pexrc_env + ) diff --git a/src/commands/inject.rs b/src/commands/inject.rs index a467591..63bfdc3 100644 --- a/src/commands/inject.rs +++ b/src/commands/inject.rs @@ -3,16 +3,18 @@ use std::collections::HashSet; use std::io; -use std::io::{Read, Write}; -use std::path::Path; +use std::io::{BufRead, Read, Write}; +use std::path::{Path, PathBuf}; -use anyhow::Context; -use boot::{inject_boot, sh_boot_shebang}; +use anyhow::{Context, anyhow}; +use boot::{inject_boot, sh_boot_shebang, write_boot}; +use fs_err as fs; use fs_err::File; use log::info; use owo_colors::OwoColorize; +use pex::{Layout, Pex}; use platform::mark_executable; -use resources::{Resources, embedded}; +use scripts::Scripts; use tempfile::NamedTempFile; use zip::write::SimpleFileOptions; use zip::{CompressionMethod, ZipArchive, ZipWriter}; @@ -22,11 +24,112 @@ use crate::clibs::CLIBS_DIR; pub fn inject( pex: &Path, compression_level: Option, - clibs: Option>, + clibs: Option<&HashSet<&Path>>, +) -> anyhow::Result<()> { + let pex = Pex::load(pex)?; + match pex.layout { + Layout::Loose => todo!("Loose PEX injection."), + Layout::Packed => inject_packed_pex(pex.path, clibs), + Layout::ZipApp => inject_pex_zip(pex.path, compression_level, clibs), + } +} + +fn inject_packed_pex(pex: &Path, clibs: Option<&HashSet<&Path>>) -> anyhow::Result<()> { + // Make sure we have a shebang early. This partially validates the pex to inject is a valid one + // before expending too much effort copying files below. + let shebang = if let Some(sh_boot_shebang) = sh_boot_shebang(pex, true)? { + sh_boot_shebang + } else { + let original_main = pex.join("__main__.py"); + io::BufReader::new(File::open(&original_main)?) + .lines() + .next() + .ok_or_else(|| { + anyhow!( + "Expected original PEX __main__.py to have a shebang line but {path} did not.", + path = original_main.display() + ) + })?? + }; + + let mut dest_pex = tempfile::tempdir_in(pex.parent().unwrap_or_else(|| Path::new(".")))?; + let excludes: HashSet = [ + ".bootstrap", + "__main__.py", + "__pex__", + "__pycache__", + "pex", + "pex-repl", + ] + .into_iter() + .map(|rel_path| pex.join(rel_path)) + .collect(); + for entry in walkdir::WalkDir::new(pex) + .min_depth(1) + .into_iter() + .filter_entry(|entry| !excludes.contains(entry.path())) + { + let entry = entry?; + let dst = dest_pex.path().join(entry.path().strip_prefix(pex)?); + if entry.path().is_dir() { + fs::create_dir_all(dst)?; + } else { + fs::copy(entry.path(), dst)?; + } + } + + let mut resources = Scripts::Embedded; + let pex_dir = dest_pex.path().join("__pex__"); + fs::create_dir_all(&pex_dir)?; + resources.write_scripts(dest_pex.path())?; + + let dst = pex.with_extension("pexrc"); + let clib_dir = pex_dir.join(".clib"); + fs::create_dir_all(&clib_dir)?; + + info!("Embedded clibs:"); + for file in CLIBS_DIR.files() { + let path = file.path(); + if let Some(clibs) = clibs.as_ref() + && !clibs.contains(path) + { + continue; + } + + let dst_path = clib_dir.join(path); + anstream::eprint!( + "Writing {entry} {size} bytes to {dst_path}...", + entry = path.display().blue(), + size = file.contents().len(), + dst_path = dst.join("__pex__").join(".clib").join(path).display(), + ); + let mut dst_file = File::create_new(dst_path)?; + let mut clib_reader = zstd::Decoder::new(file.contents())?; + io::copy(&mut clib_reader, &mut dst_file)?; + anstream::eprintln!("{}.", "done".green()) + } + + write_boot(dest_pex.path(), &shebang)?; + Layout::Packed.record(dest_pex.path())?; + + if dst.is_dir() { + fs::remove_dir_all(&dst)?; + } else if dst.is_file() { + fs::remove_file(&dst)?; + } + fs::rename(dest_pex.path(), dst)?; + dest_pex.disable_cleanup(true); + Ok(()) +} + +fn inject_pex_zip( + pex: &Path, + compression_level: Option, + clibs: Option<&HashSet<&Path>>, ) -> anyhow::Result<()> { let zip_read_fp = File::open(pex)?; let mut src_zip = ZipArchive::new(&zip_read_fp)?; - let prefix = if let Some(sh_boot_shebang) = sh_boot_shebang(pex)? { + let prefix = if let Some(sh_boot_shebang) = sh_boot_shebang(pex, false)? { Some(sh_boot_shebang.into_bytes()) } else { let first_entry = src_zip.by_index(0)?; @@ -86,7 +189,7 @@ pub fn inject( } } - let mut resources = embedded::RESOURCES; + let mut resources = Scripts::Embedded; dst_zip.add_directory("__pex__", directory_options)?; resources.inject_scripts(&mut dst_zip, zstd_file_options)?; @@ -121,7 +224,12 @@ pub fn inject( dst_zip.finish()?; mark_executable(dst_zip_fp.as_file_mut())?; - dst_zip_fp.persist(pex.with_extension("pexrc"))?; + + let dst = pex.with_extension("pexrc"); + if dst.is_dir() { + fs::remove_dir_all(&dst)?; + } + dst_zip_fp.persist(dst)?; Ok(()) }