fix(probe): one METADATA parse, a marker environment, and paths with spaces
probe.py read each distribution's METADATA three times - dist.metadata, then dist.version and dist.requires, which each re-parse it - on the path every environment open pays (M42). One parse now; an .egg-info still gets its requires.txt. On CPython 3.14 with 64 distributions the probe went from 154 ms to 118 ms (median of 15, twice), and the dists array is byte-identical. Probe schema 3 adds the PEP 508 marker environment, so the graph can tell a requirement in force from one gated to another platform (H25, consumed in the next commit). It is built from sys and os: platform.uname() is a WMI query on Windows (55 ms on 3.14) for two values almost no requirement mentions, which are left out there and stay unevaluated. Discovery: - py -0p and uv python list rows are "<key> [*] <path>", and the path may contain spaces (H26). Taking the last token turned C:\Program Files\Python312\python.exe - where every all-users python.org install goes - into a relative path the existence check dropped. - A 32-bit all-users Python registers under Wow6432Node, which a 64-bit process does not see through the default registry view (M48). HKLM is now read in both views. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c0301fa3e8
commit
53dedf8625
2 files changed
+190
-18
No files matched your search
@@ -22,7 +22,7 @@ use crate::errors::{Code, PdError, Result};
|
||||
use crate::exec::{Command, TempFile, canonical_interpreter, watchdog};
|
||||
use crate::model::{Dist, EnvSource, PkgName, PyEnv, Version};
|
||||
|
||||
/// The embedded introspection helper, executed as `<env-python> -I probe.py --json`.
|
||||
/// The embedded introspection helper, executed as `<env-python> -X utf8 -I probe.py --json`.
|
||||
///
|
||||
/// SECURITY §2: `-I` (isolated mode) makes the probe ignore `PYTHONPATH` and user site, so a
|
||||
/// poisoned environment cannot inject code into it. It is written to a temp file with a random
|
||||
@@ -39,6 +39,10 @@ pub struct ProbeResult {
|
||||
pub env: PyEnv,
|
||||
/// Installed distributions, sorted by normalized name.
|
||||
pub dists: Vec<Dist>,
|
||||
/// The PEP 508 marker environment (`sys_platform`, `platform_machine`, …), from probe schema
|
||||
/// 3. Empty for an older probe, which leaves those markers unevaluated — the conservative
|
||||
/// direction (`graph::markers`).
|
||||
pub markers: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
/// Identity of an environment: SHA-256 of the canonicalized interpreter path (ARCHITECTURE §6).
|
||||
@@ -187,7 +191,21 @@ pub fn parse_probe(stdout: &str, interpreter: &Path, source: EnvSource) -> Resul
|
||||
source,
|
||||
};
|
||||
|
||||
Ok(ProbeResult { env, dists })
|
||||
let markers = doc
|
||||
.get("markers")
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.map(|m| {
|
||||
m.iter()
|
||||
.filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_owned())))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(ProbeResult {
|
||||
env,
|
||||
dists,
|
||||
markers,
|
||||
})
|
||||
}
|
||||
|
||||
/// A discovered interpreter, before it has been probed.
|
||||
@@ -296,30 +314,39 @@ pub async fn scan_reporting(report: &(dyn Fn(ScanProgress) + Sync)) -> Vec<Candi
|
||||
found.into_values().collect()
|
||||
}
|
||||
|
||||
/// PEP 514 registry entries, from both `HKCU` and `HKLM`.
|
||||
/// PEP 514 registry entries: `HKCU`, and both views of `HKLM`.
|
||||
///
|
||||
/// SP-6 found `PythonCore` present under **both** hives on the reference machine, which is one of
|
||||
/// the two ways the same interpreter reaches discovery twice.
|
||||
///
|
||||
/// PEP 514 puts a 32-bit all-users install under `HKLM\Software\Wow6432Node\Python`, which a
|
||||
/// 64-bit process does not see through the default view — so every such Python was missing from
|
||||
/// the registry source. `HKCU` is shared between the views and needs no second read.
|
||||
#[must_use]
|
||||
pub fn registry_interpreters() -> Vec<PathBuf> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use winreg::RegKey;
|
||||
use winreg::enums::{HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE, KEY_READ};
|
||||
use winreg::enums::{HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_32KEY};
|
||||
|
||||
let mut out = Vec::new();
|
||||
for hive in [HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE] {
|
||||
for (hive, view) in [
|
||||
(HKEY_CURRENT_USER, 0),
|
||||
(HKEY_LOCAL_MACHINE, 0),
|
||||
(HKEY_LOCAL_MACHINE, KEY_WOW64_32KEY),
|
||||
] {
|
||||
let flags = KEY_READ | view;
|
||||
let root = RegKey::predef(hive);
|
||||
let Ok(software) = root.open_subkey_with_flags(r"SOFTWARE\Python", KEY_READ) else {
|
||||
let Ok(software) = root.open_subkey_with_flags(r"SOFTWARE\Python", flags) else {
|
||||
continue;
|
||||
};
|
||||
for company in software.enum_keys().flatten() {
|
||||
let Ok(company_key) = software.open_subkey_with_flags(&company, KEY_READ) else {
|
||||
let Ok(company_key) = software.open_subkey_with_flags(&company, flags) else {
|
||||
continue;
|
||||
};
|
||||
for tag in company_key.enum_keys().flatten() {
|
||||
let Ok(install) =
|
||||
company_key.open_subkey_with_flags(format!(r"{tag}\InstallPath"), KEY_READ)
|
||||
company_key.open_subkey_with_flags(format!(r"{tag}\InstallPath"), flags)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
@@ -377,13 +404,25 @@ pub async fn uv_interpreters() -> Vec<PathBuf> {
|
||||
out.stdout.lines().filter_map(parse_listed_path).collect()
|
||||
}
|
||||
|
||||
/// Take the trailing path from a listing line, rejecting `<download available>` placeholders.
|
||||
/// The interpreter path of one `py -0p` or `uv python list` row.
|
||||
///
|
||||
/// Both print a key that never contains whitespace — `-V:3.14`, `cpython-3.14.7-windows-x86_64-none`
|
||||
/// — then, for `py`, an optional `*` marking the default, then the path. **The path may contain
|
||||
/// spaces.** Taking the last whitespace-separated token turned `C:\Program Files\Python312\python.exe`
|
||||
/// — where every all-users python.org installer puts it — into the relative
|
||||
/// `Files\Python312\python.exe`, which the existence check then quietly dropped. A row uv offers to
|
||||
/// download carries `<download available>` where the path would be, and is rejected.
|
||||
fn parse_listed_path(line: &str) -> Option<PathBuf> {
|
||||
let last = line.split_whitespace().next_back()?;
|
||||
if last.starts_with('<') || !last.contains(std::path::MAIN_SEPARATOR) {
|
||||
let (_key, rest) = line.trim().split_once(char::is_whitespace)?;
|
||||
let rest = rest.trim_start();
|
||||
let rest = rest.strip_prefix('*').map_or(rest, str::trim_start);
|
||||
// uv prints a symlink as `path -> target`; the listed path is the interpreter itself.
|
||||
let path = rest.split(" -> ").next()?.trim_end();
|
||||
if path.is_empty() || path.starts_with('<') {
|
||||
return None;
|
||||
}
|
||||
Some(PathBuf::from(last))
|
||||
let path = PathBuf::from(path);
|
||||
path.is_absolute().then_some(path)
|
||||
}
|
||||
|
||||
/// Virtual environments in `dir`, one level deep.
|
||||
@@ -576,6 +615,78 @@ mod tests {
|
||||
assert!(parse_listed_path("").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_listed_path_with_spaces_survives() {
|
||||
// Where every all-users python.org installer puts Python. The last token alone was the
|
||||
// relative `Files\Python312\python.exe`, and the interpreter vanished from discovery.
|
||||
let program_files = r"C:\Program Files\Python312\python.exe";
|
||||
for line in [
|
||||
format!(" -V:3.12 {program_files}"),
|
||||
format!(" -V:3.12 * {program_files}\r"),
|
||||
format!(" -3.12-64 {program_files}"),
|
||||
format!("cpython-3.12.10-windows-x86_64-none {program_files}"),
|
||||
] {
|
||||
assert_eq!(
|
||||
parse_listed_path(&line),
|
||||
Some(PathBuf::from(program_files)),
|
||||
"{line:?}"
|
||||
);
|
||||
}
|
||||
// uv's warnings, should one reach stdout, are not paths.
|
||||
assert!(
|
||||
parse_listed_path("warning: Failed to inspect Python interpreter from search path")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
parse_listed_path("cpython-3.16.0a1-windows-x86_64-none <download available>")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_marker_environment_is_carried_and_an_older_probe_has_none() {
|
||||
let doc = r#"{"schema":3,"python":"3.14.7","prefix":"C:\\Python314",
|
||||
"externally_managed":false,"hidden_user_site":null,
|
||||
"markers":{"sys_platform":"win32","platform_machine":"AMD64","bogus":7},
|
||||
"dists":[]}"#;
|
||||
let got = parse_probe(doc, Path::new("python.exe"), EnvSource::Registry).expect("parse");
|
||||
assert_eq!(
|
||||
got.markers.get("sys_platform").map(String::as_str),
|
||||
Some("win32")
|
||||
);
|
||||
assert_eq!(
|
||||
got.markers.get("platform_machine").map(String::as_str),
|
||||
Some("AMD64")
|
||||
);
|
||||
assert!(
|
||||
!got.markers.contains_key("bogus"),
|
||||
"only strings are markers"
|
||||
);
|
||||
|
||||
let old = r#"{"schema":2,"python":"3.12.4","prefix":"x","externally_managed":false,
|
||||
"hidden_user_site":null,"dists":[]}"#;
|
||||
let got = parse_probe(old, Path::new("python.exe"), EnvSource::Registry).expect("parse");
|
||||
assert!(got.markers.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_probe_reads_metadata_once_per_distribution() {
|
||||
// `dist.version` and `dist.requires` each re-parse METADATA; on the path every environment
|
||||
// open pays, asking all three tripled the work. Measured on CPython 3.14 with 64
|
||||
// distributions: 154 ms before, 118 ms after, the same document either way.
|
||||
// On the code, not the words: the probe's own comment names both on purpose.
|
||||
assert!(
|
||||
PROBE_PY.contains(r#""version": meta.get("Version")"#),
|
||||
"the version comes from the one parse"
|
||||
);
|
||||
assert!(
|
||||
PROBE_PY.contains(r#"meta.get_all("Requires-Dist")"#),
|
||||
"the requirements come from the one parse"
|
||||
);
|
||||
// And it does not reach for `platform`, whose uname() is a WMI query on Windows.
|
||||
assert!(!PROBE_PY.contains("import platform"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_probe_python_floor_matches_the_document() {
|
||||
assert_eq!(MIN_PROBE_PYTHON, (3, 10));
|
||||
|
||||
@@ -4,7 +4,7 @@ Prints a single JSON document describing the interpreter it runs under and every
|
||||
distribution installed in it. Rust cannot cheaply read a foreign environment's
|
||||
metadata, so this helper is executed as:
|
||||
|
||||
<env-python> -I <tempfile> --json
|
||||
<env-python> -X utf8 -I <tempfile> --json
|
||||
|
||||
Constraints (docs/ARCHITECTURE.md section 4, docs/SECURITY.md section 2):
|
||||
|
||||
@@ -12,12 +12,15 @@ Constraints (docs/ARCHITECTURE.md section 4, docs/SECURITY.md section 2):
|
||||
runs in are the user's, and may contain anything.
|
||||
* Python 3.10 floor: `importlib.metadata` only, no `packaging`.
|
||||
* Run with -I so PYTHONPATH and user site are ignored and a poisoned environment
|
||||
cannot inject code into the probe.
|
||||
cannot inject code into the probe. -I also ignores PYTHONUTF8, which is why the
|
||||
core passes -X utf8: this prints paths unescaped, and a venv under a folder
|
||||
named in Vietnamese crashed the probe on the ANSI code page.
|
||||
* Written to a temp file with a random name per invocation; never installed.
|
||||
|
||||
Output shape:
|
||||
|
||||
{"python": "3.12.4", "prefix": "...", "externally_managed": false,
|
||||
{"schema": 3, "python": "3.12.4", "prefix": "...", "externally_managed": false,
|
||||
"markers": {"sys_platform": "win32", "python_version": "3.12", ...},
|
||||
"dists": [{"name": "requests", "version": "2.32.3",
|
||||
"requires_dist": ["urllib3<3,>=1.21.1"],
|
||||
"requires_python": ">=3.8", "size_bytes": 131072}]}
|
||||
@@ -34,7 +37,7 @@ import os
|
||||
import sys
|
||||
import sysconfig
|
||||
|
||||
SCHEMA_VERSION = 2
|
||||
SCHEMA_VERSION = 3
|
||||
|
||||
|
||||
def _normalize(name: str) -> str:
|
||||
@@ -86,6 +89,55 @@ def _externally_managed() -> bool:
|
||||
return any(os.path.isfile(p) for p in candidates)
|
||||
|
||||
|
||||
def _full_version(info) -> str:
|
||||
"""`3.12.4`, or `3.14.0rc1` -- PEP 508's spelling of a version_info."""
|
||||
version = f"{info.major}.{info.minor}.{info.micro}"
|
||||
if info.releaselevel != "final":
|
||||
version += info.releaselevel[0] + str(info.serial)
|
||||
return version
|
||||
|
||||
|
||||
def _marker_env() -> dict[str, str]:
|
||||
"""The PEP 508 environment this interpreter evaluates requirement markers against.
|
||||
|
||||
The core needs it to tell an in-force requirement from one gated to another platform:
|
||||
without `sys_platform`, ipython's `pexpect; sys_platform != "win32"` read as a missing
|
||||
dependency of every Windows environment that had ipython.
|
||||
|
||||
Built from `sys` and `os`, not `platform`. On Windows `platform.uname()` runs a WMI query
|
||||
-- 55 ms on 3.14, a third of a small environment's whole probe -- for two values,
|
||||
`platform_release` and `platform_version`, that almost no requirement mentions. Those two
|
||||
are therefore omitted on Windows, and the core keeps any requirement whose marker it cannot
|
||||
evaluate. `platform_machine` reads the same PROCESSOR_* variables `platform` itself falls
|
||||
back to there.
|
||||
"""
|
||||
env = {
|
||||
"implementation_name": sys.implementation.name,
|
||||
"implementation_version": _full_version(sys.implementation.version),
|
||||
"os_name": os.name,
|
||||
"python_full_version": _full_version(sys.version_info),
|
||||
"python_version": f"{sys.version_info[0]}.{sys.version_info[1]}",
|
||||
"sys_platform": sys.platform,
|
||||
}
|
||||
implementation = {"cpython": "CPython", "pypy": "PyPy"}.get(sys.implementation.name)
|
||||
if implementation:
|
||||
env["platform_python_implementation"] = implementation
|
||||
if os.name == "nt":
|
||||
env["platform_system"] = "Windows"
|
||||
machine = os.environ.get("PROCESSOR_ARCHITEW6432") or os.environ.get(
|
||||
"PROCESSOR_ARCHITECTURE"
|
||||
)
|
||||
if machine:
|
||||
env["platform_machine"] = machine
|
||||
elif hasattr(os, "uname"):
|
||||
uname = os.uname()
|
||||
env["platform_system"] = uname.sysname
|
||||
env["platform_release"] = uname.release
|
||||
env["platform_version"] = uname.version
|
||||
env["platform_machine"] = uname.machine
|
||||
return env
|
||||
|
||||
|
||||
def _hidden_user_site() -> str | None:
|
||||
"""Return the user site-packages path when it exists but is NOT on this run's sys.path.
|
||||
|
||||
@@ -204,6 +256,9 @@ def _dists() -> list[dict[str, object]]:
|
||||
seen: set[str] = set()
|
||||
for dist in distributions():
|
||||
try:
|
||||
# **One parse of METADATA per distribution.** `dist.version` and `dist.requires` each
|
||||
# re-read and re-parse it, so asking all three cost three parses per package on the
|
||||
# path every environment open pays.
|
||||
meta = dist.metadata
|
||||
if meta is None:
|
||||
continue
|
||||
@@ -218,11 +273,16 @@ def _dists() -> list[dict[str, object]]:
|
||||
if name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
requires = meta.get_all("Requires-Dist")
|
||||
if requires is None and dist.read_text("requires.txt") is not None:
|
||||
# An .egg-info keeps its requirements in requires.txt, in sections
|
||||
# importlib.metadata turns into markers; only then is the second read needed.
|
||||
requires = dist.requires
|
||||
out.append(
|
||||
{
|
||||
"name": name,
|
||||
"version": dist.version or "",
|
||||
"requires_dist": list(dist.requires or []),
|
||||
"version": meta.get("Version") or "",
|
||||
"requires_dist": list(requires or []),
|
||||
"requires_python": meta.get("Requires-Python"),
|
||||
"size_bytes": _size_bytes(dist),
|
||||
}
|
||||
@@ -253,6 +313,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"executable": sys.executable,
|
||||
"is_venv": sys.prefix != sys.base_prefix,
|
||||
"externally_managed": _externally_managed(),
|
||||
"markers": _marker_env(),
|
||||
# Non-null means -I is hiding packages that pip would list; the UI shows a note.
|
||||
"hidden_user_site": _hidden_user_site(),
|
||||
"dists": _dists(),
|
||||
|
||||
Reference in new issue
Block a user