From 80b2cee9e7047ede340df892017a9d98b764dba8 Mon Sep 17 00:00:00 2001 From: poli0981 <127664709+poli0981@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:47:04 +0700 Subject: [PATCH] fix(engine): requirements and constraints reach the engine as files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The command line no longer grows with the environment.** Every plan restates the whole installed set as requirements (SP-1), and it did so as one argv entry per package — at about 1,300 packages that passes Windows' 32,767-character command line, the spawn fails with os error 206, and the user was told to install an engine. Resolves and installs now pass `-r ` (and `-c `), written under fresh random names with `TempFile` and deleted on drop. The files hold exactly the tokens argv held: rendered `PinnedSpec`s and validated `Spec`s. A test restates 2,000 long names and asserts the command line stays under 1,000 characters. **Every install step holds the rest of the environment still.** SP-1's rule was applied to the resolve only. Phase B then ran `pip install alpha==2` bare, free to move any dependency — pip warns and exits 0 when it breaks an installed package's bound — so a failed batch could leave a state the preview never showed. `Engine::install` now takes an `InstallSet { specs, constraints }`: Phase A constrains everything the plan does not touch to its current version; a Phase B step adds the rest of the plan at its planned versions; a rollback restore holds the rest of the restore at the snapshot's. Verified against real engines in a throwaway venv (pip 26.2.1 and uv 0.13.1): `-r`/`-c` dry runs, the SP-1 held-back case and a real install all behave as before. Co-Authored-By: Claude Opus 5.5 --- crates/pipdock-core/src/engine/mod.rs | 122 ++++++++++++++++++++++++- crates/pipdock-core/src/engine/pip.rs | 48 +++++----- crates/pipdock-core/src/engine/uv.rs | 20 ++-- crates/pipdock-core/src/plan/mod.rs | 58 +++++++++++- crates/pipdock-core/tests/execution.rs | 60 +++++++++++- 5 files changed, 269 insertions(+), 39 deletions(-) diff --git a/crates/pipdock-core/src/engine/mod.rs b/crates/pipdock-core/src/engine/mod.rs index c10d1e3..f593ec5 100644 --- a/crates/pipdock-core/src/engine/mod.rs +++ b/crates/pipdock-core/src/engine/mod.rs @@ -212,14 +212,14 @@ pub trait Engine: Send + Sync { /// without a report accepted in the same session. async fn resolve(&self, env: &PyEnv, req: &PlanRequest) -> Result; - /// Install an exact pinned set. + /// Install an exact pinned set, holding everything in `set.constraints` where it is. /// /// `Batch` mode passes the whole set in one invocation (Phase A); `Isolated` mode is the /// per-package retry loop (Phase B), where a failure must **not** stop the caller's loop. async fn install( &self, env: &PyEnv, - specs: &[PinnedSpec], + set: &InstallSet, mode: ExecMode, sink: ProgressSink, ) -> Result; @@ -249,6 +249,91 @@ pub trait Engine: Send + Sync { async fn upgrade_pip(&self, env: &PyEnv) -> Result; } +/// One install invocation: what to install, and what must not move while it happens. +/// +/// SP-1's rule — restate the installed set or the resolver breaks it — was applied to the +/// *resolve* only. Phase B then installed each package as a bare `pip install name==v`, with no +/// guard: free to pull a dependency past an installed package's upper bound, which pip reports as +/// a warning and exits 0 on. `constraints` carries the target state of everything else, passed as +/// `-c`, which constrains without installing. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct InstallSet { + /// The exact pins to install. + pub specs: Vec, + /// Everything that must stay at these versions if the engine touches it at all. + pub constraints: Vec, +} + +impl InstallSet { + /// Install `specs`, constraining nothing else. + #[must_use] + pub const fn only(specs: Vec) -> Self { + Self { + specs, + constraints: Vec::new(), + } + } +} + +/// The requirements (and constraints) files one engine invocation reads, deleted when dropped. +/// +/// **Files, not argv.** Every plan restates the whole installed set (SP-1), one argv entry per +/// package — and at around 1,300 packages that passes Windows' 32,767-character command line, so +/// the spawn failed and the user was told to install an engine (PD-ENG-001). `-r`/`-c` keep the +/// command line a constant length. The files hold only rendered [`PinnedSpec`]s and validated +/// [`crate::model::Spec`]s — the same tokens argv held — written under fresh random names. +#[derive(Debug)] +pub struct RequirementFiles { + requirements: crate::exec::TempFile, + constraints: Option, +} + +impl RequirementFiles { + /// Write `requirements` (one per line) and, when there are any, `constraints`. + /// + /// # Errors + /// As [`crate::exec::TempFile::create`]. + pub fn write(requirements: &[String], constraints: &[PinnedSpec]) -> Result { + let lines = |items: &mut dyn Iterator| { + let mut text = items.collect::>().join("\n"); + text.push('\n'); + text + }; + let requirements = crate::exec::TempFile::create( + "pipdock-req", + "txt", + &lines(&mut requirements.iter().cloned()), + )?; + let constraints = if constraints.is_empty() { + None + } else { + Some(crate::exec::TempFile::create( + "pipdock-con", + "txt", + &lines(&mut constraints.iter().map(PinnedSpec::to_requirement)), + )?) + }; + Ok(Self { + requirements, + constraints, + }) + } + + /// `-r [-c ]`, one argv entry each. + #[must_use] + pub fn args(&self) -> Vec { + let mut out = vec![ + "-r".to_owned(), + self.requirements.path().display().to_string(), + ]; + if let Some(c) = &self.constraints { + out.push("-c".to_owned()); + out.push(c.path().display().to_string()); + } + out + } +} + /// The adapter for `id`. /// /// Both heads reach this point by different routes — the CLI from `--engine` or the stored @@ -609,4 +694,37 @@ mod tests { assert_eq!(guards.len(), 1); assert_eq!(guards[0].to_requirement(), "good==1.0"); } + + #[test] + fn a_huge_environment_still_fits_on_a_command_line() { + // Every plan restates the installed set (SP-1). As argv, ~1,300 packages passed Windows' + // 32,767-character limit, the spawn failed with os error 206, and the user was told to + // install an engine. As a file, the command line no longer grows with the environment. + let installed: Vec = (0..2000) + .map(|i| crate::model::Dist { + name: PkgName::parse(&format!("package-with-a-long-name-{i:04}")).expect("name"), + version: crate::model::Version("12.34.5678".into()), + requires_dist: Vec::new(), + requires_python: None, + size_bytes: None, + }) + .collect(); + let req = PlanRequest { + upgrades: Vec::new(), + installs: Vec::new(), + strategy: crate::plan::Strategy::Compatible, + }; + let lines = plan_argv_specs(&req, &installed); + assert!( + lines.iter().map(|l| l.len() + 1).sum::() > 32_767, + "the fixture must be big enough to have broken argv" + ); + let files = RequirementFiles::write(&lines, &[]).expect("written"); + let argv = pip::PipEngine::argv_dry_run(&files.args()); + let command_line: usize = argv.iter().map(|a| a.len() + 1).sum(); + assert!( + command_line < 1_000, + "command line is {command_line} characters" + ); + } } diff --git a/crates/pipdock-core/src/engine/pip.rs b/crates/pipdock-core/src/engine/pip.rs index a79ab5d..add8214 100644 --- a/crates/pipdock-core/src/engine/pip.rs +++ b/crates/pipdock-core/src/engine/pip.rs @@ -14,7 +14,7 @@ use crate::model::{ }; use crate::plan::{PlanRequest, ResolutionReport}; -use super::{Engine, ProgressSink, single_pkg}; +use super::{Engine, InstallSet, ProgressSink, single_pkg}; /// Drives pip as a subprocess. #[derive(Debug, Clone, Copy, Default)] @@ -46,10 +46,10 @@ impl PipEngine { /// argv for the dry-run resolve, whose JSON report goes to stdout. /// - /// `requirements` comes from [`super::plan_argv_specs`] and carries all three groups: bare - /// names to move, explicit installs, and the pinned guard set. + /// `files` is [`super::RequirementFiles::args`] over [`super::plan_argv_specs`], which carries + /// all three groups: bare names to move, explicit installs, and the pinned guard set. #[must_use] - pub fn argv_dry_run(requirements: &[String]) -> Vec { + pub fn argv_dry_run(files: &[String]) -> Vec { let mut argv = vec![ "-m".into(), "pip".into(), @@ -64,7 +64,7 @@ impl PipEngine { "--report".into(), "-".into(), ]; - argv.extend(requirements.iter().cloned()); + argv.extend(files.iter().cloned()); argv } @@ -137,9 +137,9 @@ impl Engine for PipEngine { // SP-1: the installed set must be restated or the resolver ignores it and plans an // upgrade that breaks installed dependents. The caller assembles that set into `req`. let installed = self.list_installed(env).await?; - let requirements = super::plan_argv_specs(req, &installed); + let files = super::RequirementFiles::write(&super::plan_argv_specs(req, &installed), &[])?; let out = Command::python(&env.interpreter) - .args(Self::argv_dry_run(&requirements)) + .args(Self::argv_dry_run(&files.args())) .timeout(watchdog::RESOLVE) .run() .await?; @@ -152,19 +152,21 @@ impl Engine for PipEngine { async fn install( &self, env: &PyEnv, - specs: &[PinnedSpec], + set: &InstallSet, mode: ExecMode, sink: ProgressSink, ) -> Result { + let requirements: Vec = set.specs.iter().map(PinnedSpec::to_requirement).collect(); + let files = super::RequirementFiles::write(&requirements, &set.constraints)?; let mut argv = vec!["-m".to_owned(), "pip".to_owned(), "install".to_owned()]; - argv.extend(specs.iter().map(PinnedSpec::to_requirement)); + argv.extend(files.args()); let out = Command::python(&env.interpreter) .args(argv) .timeout(watchdog::INSTALL) .cancel(sink.cancel.clone()) - .run_streaming(&sink, single_pkg(specs), mode) + .run_streaming(&sink, single_pkg(&set.specs), mode) .await?; - Ok(super::step_result(specs, &out)) + Ok(super::step_result(&set.specs, &out)) } async fn uninstall( @@ -341,8 +343,9 @@ mod tests { } #[test] - fn requirements_are_appended_as_separate_argv_entries() { - // One token per argument is what makes shell quoting irrelevant (SECURITY §2). + fn requirements_reach_pip_as_a_file_not_as_argv() { + // A requirements file keeps the command line a constant length whatever the size of the + // environment, and each path is still exactly one argv entry (SECURITY §2). let specs = [ PinnedSpec { name: PkgName::parse("httpx").unwrap(), @@ -354,15 +357,16 @@ mod tests { }, ]; let requirements: Vec = specs.iter().map(PinnedSpec::to_requirement).collect(); - let argv = PipEngine::argv_dry_run(&requirements); - assert_eq!( - &argv[argv.len() - 2..], - ["httpx==0.28.1", "requests==2.32.3"] - ); - assert!( - argv.iter().all(|a| !a.contains(' ')), - "no argv entry may bundle multiple arguments" - ); + let files = super::super::RequirementFiles::write(&requirements, &specs[..1]) + .expect("files are written"); + let argv = PipEngine::argv_dry_run(&files.args()); + assert_eq!(&argv[argv.len() - 4..argv.len() - 3], ["-r"]); + assert_eq!(&argv[argv.len() - 2..argv.len() - 1], ["-c"]); + let written = + std::fs::read_to_string(&argv[argv.len() - 3]).expect("requirements file exists"); + assert_eq!(written, "httpx==0.28.1\nrequests==2.32.3\n"); + let constrained = std::fs::read_to_string(&argv[argv.len() - 1]).expect("constraints"); + assert_eq!(constrained, "httpx==0.28.1\n"); } #[test] diff --git a/crates/pipdock-core/src/engine/uv.rs b/crates/pipdock-core/src/engine/uv.rs index dbb5c31..0585106 100644 --- a/crates/pipdock-core/src/engine/uv.rs +++ b/crates/pipdock-core/src/engine/uv.rs @@ -25,7 +25,7 @@ use crate::model::{ }; use crate::plan::{PlanRequest, ResolutionReport}; -use super::{Engine, ProgressSink, single_pkg}; +use super::{Engine, InstallSet, ProgressSink, single_pkg}; /// Drives uv as a subprocess. #[derive(Debug, Clone, Copy, Default)] @@ -49,7 +49,7 @@ impl UvEngine { /// `requirements` comes from [`super::plan_argv_specs`] and carries all three groups: bare /// names to move, explicit installs, and the pinned guard set. #[must_use] - pub fn argv_dry_run(python: &str, requirements: &[String]) -> Vec { + pub fn argv_dry_run(python: &str, files: &[String]) -> Vec { let mut argv = vec![ "pip".into(), "install".into(), @@ -58,7 +58,7 @@ impl UvEngine { "--python".into(), python.into(), ]; - argv.extend(requirements.iter().cloned()); + argv.extend(files.iter().cloned()); argv } @@ -142,10 +142,10 @@ impl Engine for UvEngine { // SP-1: without the installed set restated, uv plans an upgrade that breaks installed // dependents and exits 0. Identical requirement to pip's adapter. let installed = self.list_installed(env).await?; - let requirements = super::plan_argv_specs(req, &installed); + let files = super::RequirementFiles::write(&super::plan_argv_specs(req, &installed), &[])?; let python = env.interpreter.display().to_string(); let out = Command::new("uv") - .args(Self::argv_dry_run(&python, &requirements)) + .args(Self::argv_dry_run(&python, &files.args())) .timeout(watchdog::RESOLVE) .run() .await?; @@ -157,25 +157,27 @@ impl Engine for UvEngine { async fn install( &self, env: &PyEnv, - specs: &[PinnedSpec], + set: &InstallSet, mode: ExecMode, sink: ProgressSink, ) -> Result { let python = env.interpreter.display().to_string(); + let requirements: Vec = set.specs.iter().map(PinnedSpec::to_requirement).collect(); + let files = super::RequirementFiles::write(&requirements, &set.constraints)?; let mut argv = vec![ "pip".to_owned(), "install".to_owned(), "--python".to_owned(), python, ]; - argv.extend(specs.iter().map(PinnedSpec::to_requirement)); + argv.extend(files.args()); let out = Command::new("uv") .args(argv) .timeout(watchdog::INSTALL) .cancel(sink.cancel.clone()) - .run_streaming(&sink, single_pkg(specs), mode) + .run_streaming(&sink, single_pkg(&set.specs), mode) .await?; - Ok(super::step_result(specs, &out)) + Ok(super::step_result(&set.specs, &out)) } async fn uninstall( diff --git a/crates/pipdock-core/src/plan/mod.rs b/crates/pipdock-core/src/plan/mod.rs index f103947..691884f 100644 --- a/crates/pipdock-core/src/plan/mod.rs +++ b/crates/pipdock-core/src/plan/mod.rs @@ -8,7 +8,7 @@ pub use preview::{ Decision, ForcedPlan, apply_decisions, default_decision, derive_held_back, forced_requirements, }; -use crate::engine::{Engine, ProgressSink}; +use crate::engine::{Engine, InstallSet, ProgressSink}; use crate::errors::{Code, PdError, Result}; use crate::model::{ CheckReport, ExecMode, PinnedSpec, PkgName, PyEnv, Spec, StepResult, StepStatus, Version, @@ -546,7 +546,15 @@ pub async fn execute( // all three out of `execute` as PD-INT-001: no summary, no "cancelled", no rollback offer, and // the cancelled-Phase-A branch below unreachable outside a test whose fake returned `Ok`. let batch = engine - .install(env, &pinned, ExecMode::Batch, batch_sink.clone()) + .install( + env, + &InstallSet { + specs: pinned.clone(), + constraints: hold_still(installed_now, &pinned, None), + }, + ExecMode::Batch, + batch_sink.clone(), + ) .await; let stopped_by_user = sink.is_cancelled() || batch.as_ref().is_err_and(PdError::is_cancelled); batch_sink.finished( @@ -619,7 +627,10 @@ pub async fn execute( let step = engine .install( env, - std::slice::from_ref(spec), + &InstallSet { + specs: vec![spec.clone()], + constraints: hold_still(installed_now, &pinned, Some(&spec.name)), + }, ExecMode::Isolated, step_sink.clone(), ) @@ -663,6 +674,35 @@ pub async fn execute( Ok(summary(plan, ExecMode::Isolated, results, engine, env, cancelled).await) } +/// What must not move while part of a plan installs: every **other** planned package at the +/// version the preview showed, and everything installed that the plan does not touch at the +/// version it has now. +/// +/// Phase A passes `except: None` (the whole plan is the requirement); a Phase B step passes its +/// own package. Without this a Phase B `pip install alpha==2` was free to pull a dependency past +/// an installed package's upper bound — pip warns and exits 0 — leaving a state no preview showed. +fn hold_still( + installed: &[crate::model::Dist], + pinned: &[PinnedSpec], + except: Option<&crate::model::PkgName>, +) -> Vec { + let planned: std::collections::BTreeSet<&crate::model::PkgName> = + pinned.iter().map(|p| &p.name).collect(); + let others = pinned + .iter() + .filter(|p| except.is_some_and(|name| *name != p.name)) + .cloned(); + let untouched = installed + .iter() + .filter(|d| !planned.contains(&d.name)) + .filter(|d| crate::pep440::is_argv_safe(&d.version.0)) + .map(|d| PinnedSpec { + name: d.name.clone(), + version: d.version.clone(), + }); + others.chain(untouched).collect() +} + /// Whether retrying package by package could get further than the batch did. /// /// False for failures that are the **environment's**, not one package's: the network (PD-NET-*), @@ -771,10 +811,20 @@ pub async fn execute_rollback( let step_sink = sink.at(restore.uninstall.len() + index); step_sink.started(Some(spec.name.clone()), ExecMode::Isolated); + // The rest of the restore is the target state, so it is what this step must not move. + let constraints: Vec = restore + .install + .iter() + .filter(|other| other.name != spec.name) + .cloned() + .collect(); let step = engine .install( env, - std::slice::from_ref(spec), + &InstallSet { + specs: vec![spec.clone()], + constraints, + }, ExecMode::Isolated, step_sink.clone(), ) diff --git a/crates/pipdock-core/tests/execution.rs b/crates/pipdock-core/tests/execution.rs index ba44a16..8cf90a9 100644 --- a/crates/pipdock-core/tests/execution.rs +++ b/crates/pipdock-core/tests/execution.rs @@ -16,7 +16,7 @@ use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use pipdock_core::engine::{Engine, ProgressEvent, ProgressSink}; +use pipdock_core::engine::{Engine, InstallSet, ProgressEvent, ProgressSink}; use pipdock_core::errors::Result; use pipdock_core::model::{ CheckFinding, CheckReport, Dist, EngineId, EngineInfo, EnvSource, ExecMode, OutdatedDist, @@ -61,6 +61,8 @@ struct FakeEngine { check: CheckReport, /// How Phase A ends. batch: Batch, + /// The constraints each install was given, in call order. + constraints: Arc>>>, } impl FakeEngine { @@ -73,6 +75,7 @@ impl FakeEngine { findings: Vec::new(), }, batch: Batch::Runs, + constraints: Arc::new(Mutex::new(Vec::new())), } } @@ -115,10 +118,15 @@ impl Engine for FakeEngine { async fn install( &self, _env: &PyEnv, - specs: &[PinnedSpec], + set: &InstallSet, mode: ExecMode, _sink: ProgressSink, ) -> Result { + let specs = &set.specs; + self.constraints + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(set.constraints.clone()); let names: Vec = specs.iter().map(|s| s.name.to_string()).collect(); self.calls .lock() @@ -736,6 +744,54 @@ async fn every_row_says_what_it_moved_from() { ); } +#[tokio::test] +async fn an_isolated_step_holds_everything_else_where_the_preview_put_it() { + // SP-1's rule was applied to the resolve and nowhere else: Phase B installed each package as a + // bare `name==v`, free to move anything as a dependency. Every step now carries the rest of + // the plan at its planned versions and the untouched packages at their current ones. + let engine = FakeEngine::new(&["badone"]); + let installed = [Dist { + name: PkgName::parse("bystander").unwrap(), + version: Version("1.5".into()), + requires_dist: Vec::new(), + requires_python: None, + size_bytes: None, + }]; + let dir = scratch(); + let _ = std::fs::remove_dir_all(&dir); + let snap = snapshot_for(&dir); + let plan = AcceptedPlan::accept( + report(&[("alpha", "2.0"), ("badone", "1.0")]), + "envhash01".to_owned(), + &installed, + now(), + ); + let summary = execute( + &engine, + &env(), + &plan, + SnapshotProof::Taken(&snap), + &installed, + now(), + sink(), + ) + .await + .expect("execute"); + let _ = std::fs::remove_dir_all(&dir); + assert_eq!(summary.phase, ExecMode::Isolated); + + let rendered = |set: &[PinnedSpec]| -> Vec { + set.iter().map(PinnedSpec::to_requirement).collect() + }; + let constraints = engine.constraints.lock().unwrap().clone(); + // Phase A: the plan is the requirement, the bystander is held. + assert_eq!(rendered(&constraints[0]), ["bystander==1.5"]); + // Phase B, alpha: badone at its planned version, and the bystander. + assert_eq!(rendered(&constraints[1]), ["badone==1.0", "bystander==1.5"]); + // Phase B, badone: alpha at its planned version, and the bystander. + assert_eq!(rendered(&constraints[2]), ["alpha==2.0", "bystander==1.5"]); +} + #[tokio::test] async fn an_uncancelled_run_does_not_claim_it_was_cancelled() { let summary = run(&FakeEngine::new(&[]), &[("alpha", "1.0")]).await;