fix(engine): requirements and constraints reach the engine as files

**The command line no longer grows with the environment.** Every plan restates the whole
installed set as requirements (SP-1), and it did so as one argv entry per package — at about 1,300
packages that passes Windows' 32,767-character command line, the spawn fails with os error 206,
and the user was told to install an engine. Resolves and installs now pass `-r <file>` (and
`-c <file>`), written under fresh random names with `TempFile` and deleted on drop. The files hold
exactly the tokens argv held: rendered `PinnedSpec`s and validated `Spec`s. A test restates 2,000
long names and asserts the command line stays under 1,000 characters.

**Every install step holds the rest of the environment still.** SP-1's rule was applied to the
resolve only. Phase B then ran `pip install alpha==2` bare, free to move any dependency — pip warns
and exits 0 when it breaks an installed package's bound — so a failed batch could leave a state
the preview never showed. `Engine::install` now takes an `InstallSet { specs, constraints }`:
Phase A constrains everything the plan does not touch to its current version; a Phase B step adds
the rest of the plan at its planned versions; a rollback restore holds the rest of the restore at
the snapshot's.

Verified against real engines in a throwaway venv (pip 26.2.1 and uv 0.13.1): `-r`/`-c` dry
runs, the SP-1 held-back case and a real install all behave as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 committed 2026-10-17 00:55:27 +07:00
1 parent 6559ebcc84
commit 80b2cee9e7
5 files changed
+269 -39

No files matched your search

+120 -2
View File
@@ -212,14 +212,14 @@ pub trait Engine: Send + Sync {
/// without a report accepted in the same session.
async fn resolve(&self, env: &PyEnv, req: &PlanRequest) -> Result<ResolutionReport>;
/// Install an exact pinned set.
/// Install an exact pinned set, holding everything in `set.constraints` where it is.
///
/// `Batch` mode passes the whole set in one invocation (Phase A); `Isolated` mode is the
/// per-package retry loop (Phase B), where a failure must **not** stop the caller's loop.
async fn install(
&self,
env: &PyEnv,
specs: &[PinnedSpec],
set: &InstallSet,
mode: ExecMode,
sink: ProgressSink,
) -> Result<StepResult>;
@@ -249,6 +249,91 @@ pub trait Engine: Send + Sync {
async fn upgrade_pip(&self, env: &PyEnv) -> Result<StepResult>;
}
/// One install invocation: what to install, and what must not move while it happens.
///
/// SP-1's rule — restate the installed set or the resolver breaks it — was applied to the
/// *resolve* only. Phase B then installed each package as a bare `pip install name==v`, with no
/// guard: free to pull a dependency past an installed package's upper bound, which pip reports as
/// a warning and exits 0 on. `constraints` carries the target state of everything else, passed as
/// `-c`, which constrains without installing.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct InstallSet {
/// The exact pins to install.
pub specs: Vec<PinnedSpec>,
/// Everything that must stay at these versions if the engine touches it at all.
pub constraints: Vec<PinnedSpec>,
}
impl InstallSet {
/// Install `specs`, constraining nothing else.
#[must_use]
pub const fn only(specs: Vec<PinnedSpec>) -> Self {
Self {
specs,
constraints: Vec::new(),
}
}
}
/// The requirements (and constraints) files one engine invocation reads, deleted when dropped.
///
/// **Files, not argv.** Every plan restates the whole installed set (SP-1), one argv entry per
/// package — and at around 1,300 packages that passes Windows' 32,767-character command line, so
/// the spawn failed and the user was told to install an engine (PD-ENG-001). `-r`/`-c` keep the
/// command line a constant length. The files hold only rendered [`PinnedSpec`]s and validated
/// [`crate::model::Spec`]s — the same tokens argv held — written under fresh random names.
#[derive(Debug)]
pub struct RequirementFiles {
requirements: crate::exec::TempFile,
constraints: Option<crate::exec::TempFile>,
}
impl RequirementFiles {
/// Write `requirements` (one per line) and, when there are any, `constraints`.
///
/// # Errors
/// As [`crate::exec::TempFile::create`].
pub fn write(requirements: &[String], constraints: &[PinnedSpec]) -> Result<Self> {
let lines = |items: &mut dyn Iterator<Item = String>| {
let mut text = items.collect::<Vec<_>>().join("\n");
text.push('\n');
text
};
let requirements = crate::exec::TempFile::create(
"pipdock-req",
"txt",
&lines(&mut requirements.iter().cloned()),
)?;
let constraints = if constraints.is_empty() {
None
} else {
Some(crate::exec::TempFile::create(
"pipdock-con",
"txt",
&lines(&mut constraints.iter().map(PinnedSpec::to_requirement)),
)?)
};
Ok(Self {
requirements,
constraints,
})
}
/// `-r <requirements> [-c <constraints>]`, one argv entry each.
#[must_use]
pub fn args(&self) -> Vec<String> {
let mut out = vec![
"-r".to_owned(),
self.requirements.path().display().to_string(),
];
if let Some(c) = &self.constraints {
out.push("-c".to_owned());
out.push(c.path().display().to_string());
}
out
}
}
/// The adapter for `id`.
///
/// Both heads reach this point by different routes — the CLI from `--engine` or the stored
@@ -609,4 +694,37 @@ mod tests {
assert_eq!(guards.len(), 1);
assert_eq!(guards[0].to_requirement(), "good==1.0");
}
#[test]
fn a_huge_environment_still_fits_on_a_command_line() {
// Every plan restates the installed set (SP-1). As argv, ~1,300 packages passed Windows'
// 32,767-character limit, the spawn failed with os error 206, and the user was told to
// install an engine. As a file, the command line no longer grows with the environment.
let installed: Vec<crate::model::Dist> = (0..2000)
.map(|i| crate::model::Dist {
name: PkgName::parse(&format!("package-with-a-long-name-{i:04}")).expect("name"),
version: crate::model::Version("12.34.5678".into()),
requires_dist: Vec::new(),
requires_python: None,
size_bytes: None,
})
.collect();
let req = PlanRequest {
upgrades: Vec::new(),
installs: Vec::new(),
strategy: crate::plan::Strategy::Compatible,
};
let lines = plan_argv_specs(&req, &installed);
assert!(
lines.iter().map(|l| l.len() + 1).sum::<usize>() > 32_767,
"the fixture must be big enough to have broken argv"
);
let files = RequirementFiles::write(&lines, &[]).expect("written");
let argv = pip::PipEngine::argv_dry_run(&files.args());
let command_line: usize = argv.iter().map(|a| a.len() + 1).sum();
assert!(
command_line < 1_000,
"command line is {command_line} characters"
);
}
}
+26 -22
View File
@@ -14,7 +14,7 @@ use crate::model::{
};
use crate::plan::{PlanRequest, ResolutionReport};
use super::{Engine, ProgressSink, single_pkg};
use super::{Engine, InstallSet, ProgressSink, single_pkg};
/// Drives pip as a subprocess.
#[derive(Debug, Clone, Copy, Default)]
@@ -46,10 +46,10 @@ impl PipEngine {
/// argv for the dry-run resolve, whose JSON report goes to stdout.
///
/// `requirements` comes from [`super::plan_argv_specs`] and carries all three groups: bare
/// names to move, explicit installs, and the pinned guard set.
/// `files` is [`super::RequirementFiles::args`] over [`super::plan_argv_specs`], which carries
/// all three groups: bare names to move, explicit installs, and the pinned guard set.
#[must_use]
pub fn argv_dry_run(requirements: &[String]) -> Vec<String> {
pub fn argv_dry_run(files: &[String]) -> Vec<String> {
let mut argv = vec![
"-m".into(),
"pip".into(),
@@ -64,7 +64,7 @@ impl PipEngine {
"--report".into(),
"-".into(),
];
argv.extend(requirements.iter().cloned());
argv.extend(files.iter().cloned());
argv
}
@@ -137,9 +137,9 @@ impl Engine for PipEngine {
// SP-1: the installed set must be restated or the resolver ignores it and plans an
// upgrade that breaks installed dependents. The caller assembles that set into `req`.
let installed = self.list_installed(env).await?;
let requirements = super::plan_argv_specs(req, &installed);
let files = super::RequirementFiles::write(&super::plan_argv_specs(req, &installed), &[])?;
let out = Command::python(&env.interpreter)
.args(Self::argv_dry_run(&requirements))
.args(Self::argv_dry_run(&files.args()))
.timeout(watchdog::RESOLVE)
.run()
.await?;
@@ -152,19 +152,21 @@ impl Engine for PipEngine {
async fn install(
&self,
env: &PyEnv,
specs: &[PinnedSpec],
set: &InstallSet,
mode: ExecMode,
sink: ProgressSink,
) -> Result<StepResult> {
let requirements: Vec<String> = set.specs.iter().map(PinnedSpec::to_requirement).collect();
let files = super::RequirementFiles::write(&requirements, &set.constraints)?;
let mut argv = vec!["-m".to_owned(), "pip".to_owned(), "install".to_owned()];
argv.extend(specs.iter().map(PinnedSpec::to_requirement));
argv.extend(files.args());
let out = Command::python(&env.interpreter)
.args(argv)
.timeout(watchdog::INSTALL)
.cancel(sink.cancel.clone())
.run_streaming(&sink, single_pkg(specs), mode)
.run_streaming(&sink, single_pkg(&set.specs), mode)
.await?;
Ok(super::step_result(specs, &out))
Ok(super::step_result(&set.specs, &out))
}
async fn uninstall(
@@ -341,8 +343,9 @@ mod tests {
}
#[test]
fn requirements_are_appended_as_separate_argv_entries() {
// One token per argument is what makes shell quoting irrelevant (SECURITY §2).
fn requirements_reach_pip_as_a_file_not_as_argv() {
// A requirements file keeps the command line a constant length whatever the size of the
// environment, and each path is still exactly one argv entry (SECURITY §2).
let specs = [
PinnedSpec {
name: PkgName::parse("httpx").unwrap(),
@@ -354,15 +357,16 @@ mod tests {
},
];
let requirements: Vec<String> = specs.iter().map(PinnedSpec::to_requirement).collect();
let argv = PipEngine::argv_dry_run(&requirements);
assert_eq!(
&argv[argv.len() - 2..],
["httpx==0.28.1", "requests==2.32.3"]
);
assert!(
argv.iter().all(|a| !a.contains(' ')),
"no argv entry may bundle multiple arguments"
);
let files = super::super::RequirementFiles::write(&requirements, &specs[..1])
.expect("files are written");
let argv = PipEngine::argv_dry_run(&files.args());
assert_eq!(&argv[argv.len() - 4..argv.len() - 3], ["-r"]);
assert_eq!(&argv[argv.len() - 2..argv.len() - 1], ["-c"]);
let written =
std::fs::read_to_string(&argv[argv.len() - 3]).expect("requirements file exists");
assert_eq!(written, "httpx==0.28.1\nrequests==2.32.3\n");
let constrained = std::fs::read_to_string(&argv[argv.len() - 1]).expect("constraints");
assert_eq!(constrained, "httpx==0.28.1\n");
}
#[test]
+11 -9
View File
@@ -25,7 +25,7 @@ use crate::model::{
};
use crate::plan::{PlanRequest, ResolutionReport};
use super::{Engine, ProgressSink, single_pkg};
use super::{Engine, InstallSet, ProgressSink, single_pkg};
/// Drives uv as a subprocess.
#[derive(Debug, Clone, Copy, Default)]
@@ -49,7 +49,7 @@ impl UvEngine {
/// `requirements` comes from [`super::plan_argv_specs`] and carries all three groups: bare
/// names to move, explicit installs, and the pinned guard set.
#[must_use]
pub fn argv_dry_run(python: &str, requirements: &[String]) -> Vec<String> {
pub fn argv_dry_run(python: &str, files: &[String]) -> Vec<String> {
let mut argv = vec![
"pip".into(),
"install".into(),
@@ -58,7 +58,7 @@ impl UvEngine {
"--python".into(),
python.into(),
];
argv.extend(requirements.iter().cloned());
argv.extend(files.iter().cloned());
argv
}
@@ -142,10 +142,10 @@ impl Engine for UvEngine {
// SP-1: without the installed set restated, uv plans an upgrade that breaks installed
// dependents and exits 0. Identical requirement to pip's adapter.
let installed = self.list_installed(env).await?;
let requirements = super::plan_argv_specs(req, &installed);
let files = super::RequirementFiles::write(&super::plan_argv_specs(req, &installed), &[])?;
let python = env.interpreter.display().to_string();
let out = Command::new("uv")
.args(Self::argv_dry_run(&python, &requirements))
.args(Self::argv_dry_run(&python, &files.args()))
.timeout(watchdog::RESOLVE)
.run()
.await?;
@@ -157,25 +157,27 @@ impl Engine for UvEngine {
async fn install(
&self,
env: &PyEnv,
specs: &[PinnedSpec],
set: &InstallSet,
mode: ExecMode,
sink: ProgressSink,
) -> Result<StepResult> {
let python = env.interpreter.display().to_string();
let requirements: Vec<String> = set.specs.iter().map(PinnedSpec::to_requirement).collect();
let files = super::RequirementFiles::write(&requirements, &set.constraints)?;
let mut argv = vec![
"pip".to_owned(),
"install".to_owned(),
"--python".to_owned(),
python,
];
argv.extend(specs.iter().map(PinnedSpec::to_requirement));
argv.extend(files.args());
let out = Command::new("uv")
.args(argv)
.timeout(watchdog::INSTALL)
.cancel(sink.cancel.clone())
.run_streaming(&sink, single_pkg(specs), mode)
.run_streaming(&sink, single_pkg(&set.specs), mode)
.await?;
Ok(super::step_result(specs, &out))
Ok(super::step_result(&set.specs, &out))
}
async fn uninstall(
+54 -4
View File
@@ -8,7 +8,7 @@ pub use preview::{
Decision, ForcedPlan, apply_decisions, default_decision, derive_held_back, forced_requirements,
};
use crate::engine::{Engine, ProgressSink};
use crate::engine::{Engine, InstallSet, ProgressSink};
use crate::errors::{Code, PdError, Result};
use crate::model::{
CheckReport, ExecMode, PinnedSpec, PkgName, PyEnv, Spec, StepResult, StepStatus, Version,
@@ -546,7 +546,15 @@ pub async fn execute(
// all three out of `execute` as PD-INT-001: no summary, no "cancelled", no rollback offer, and
// the cancelled-Phase-A branch below unreachable outside a test whose fake returned `Ok`.
let batch = engine
.install(env, &pinned, ExecMode::Batch, batch_sink.clone())
.install(
env,
&InstallSet {
specs: pinned.clone(),
constraints: hold_still(installed_now, &pinned, None),
},
ExecMode::Batch,
batch_sink.clone(),
)
.await;
let stopped_by_user = sink.is_cancelled() || batch.as_ref().is_err_and(PdError::is_cancelled);
batch_sink.finished(
@@ -619,7 +627,10 @@ pub async fn execute(
let step = engine
.install(
env,
std::slice::from_ref(spec),
&InstallSet {
specs: vec![spec.clone()],
constraints: hold_still(installed_now, &pinned, Some(&spec.name)),
},
ExecMode::Isolated,
step_sink.clone(),
)
@@ -663,6 +674,35 @@ pub async fn execute(
Ok(summary(plan, ExecMode::Isolated, results, engine, env, cancelled).await)
}
/// What must not move while part of a plan installs: every **other** planned package at the
/// version the preview showed, and everything installed that the plan does not touch at the
/// version it has now.
///
/// Phase A passes `except: None` (the whole plan is the requirement); a Phase B step passes its
/// own package. Without this a Phase B `pip install alpha==2` was free to pull a dependency past
/// an installed package's upper bound — pip warns and exits 0 — leaving a state no preview showed.
fn hold_still(
installed: &[crate::model::Dist],
pinned: &[PinnedSpec],
except: Option<&crate::model::PkgName>,
) -> Vec<PinnedSpec> {
let planned: std::collections::BTreeSet<&crate::model::PkgName> =
pinned.iter().map(|p| &p.name).collect();
let others = pinned
.iter()
.filter(|p| except.is_some_and(|name| *name != p.name))
.cloned();
let untouched = installed
.iter()
.filter(|d| !planned.contains(&d.name))
.filter(|d| crate::pep440::is_argv_safe(&d.version.0))
.map(|d| PinnedSpec {
name: d.name.clone(),
version: d.version.clone(),
});
others.chain(untouched).collect()
}
/// Whether retrying package by package could get further than the batch did.
///
/// False for failures that are the **environment's**, not one package's: the network (PD-NET-*),
@@ -771,10 +811,20 @@ pub async fn execute_rollback(
let step_sink = sink.at(restore.uninstall.len() + index);
step_sink.started(Some(spec.name.clone()), ExecMode::Isolated);
// The rest of the restore is the target state, so it is what this step must not move.
let constraints: Vec<PinnedSpec> = restore
.install
.iter()
.filter(|other| other.name != spec.name)
.cloned()
.collect();
let step = engine
.install(
env,
std::slice::from_ref(spec),
&InstallSet {
specs: vec![spec.clone()],
constraints,
},
ExecMode::Isolated,
step_sink.clone(),
)
+58 -2
View File
@@ -16,7 +16,7 @@
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use pipdock_core::engine::{Engine, ProgressEvent, ProgressSink};
use pipdock_core::engine::{Engine, InstallSet, ProgressEvent, ProgressSink};
use pipdock_core::errors::Result;
use pipdock_core::model::{
CheckFinding, CheckReport, Dist, EngineId, EngineInfo, EnvSource, ExecMode, OutdatedDist,
@@ -61,6 +61,8 @@ struct FakeEngine {
check: CheckReport,
/// How Phase A ends.
batch: Batch,
/// The constraints each install was given, in call order.
constraints: Arc<Mutex<Vec<Vec<PinnedSpec>>>>,
}
impl FakeEngine {
@@ -73,6 +75,7 @@ impl FakeEngine {
findings: Vec::new(),
},
batch: Batch::Runs,
constraints: Arc::new(Mutex::new(Vec::new())),
}
}
@@ -115,10 +118,15 @@ impl Engine for FakeEngine {
async fn install(
&self,
_env: &PyEnv,
specs: &[PinnedSpec],
set: &InstallSet,
mode: ExecMode,
_sink: ProgressSink,
) -> Result<StepResult> {
let specs = &set.specs;
self.constraints
.lock()
.unwrap_or_else(|e| e.into_inner())
.push(set.constraints.clone());
let names: Vec<String> = specs.iter().map(|s| s.name.to_string()).collect();
self.calls
.lock()
@@ -736,6 +744,54 @@ async fn every_row_says_what_it_moved_from() {
);
}
#[tokio::test]
async fn an_isolated_step_holds_everything_else_where_the_preview_put_it() {
// SP-1's rule was applied to the resolve and nowhere else: Phase B installed each package as a
// bare `name==v`, free to move anything as a dependency. Every step now carries the rest of
// the plan at its planned versions and the untouched packages at their current ones.
let engine = FakeEngine::new(&["badone"]);
let installed = [Dist {
name: PkgName::parse("bystander").unwrap(),
version: Version("1.5".into()),
requires_dist: Vec::new(),
requires_python: None,
size_bytes: None,
}];
let dir = scratch();
let _ = std::fs::remove_dir_all(&dir);
let snap = snapshot_for(&dir);
let plan = AcceptedPlan::accept(
report(&[("alpha", "2.0"), ("badone", "1.0")]),
"envhash01".to_owned(),
&installed,
now(),
);
let summary = execute(
&engine,
&env(),
&plan,
SnapshotProof::Taken(&snap),
&installed,
now(),
sink(),
)
.await
.expect("execute");
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(summary.phase, ExecMode::Isolated);
let rendered = |set: &[PinnedSpec]| -> Vec<String> {
set.iter().map(PinnedSpec::to_requirement).collect()
};
let constraints = engine.constraints.lock().unwrap().clone();
// Phase A: the plan is the requirement, the bystander is held.
assert_eq!(rendered(&constraints[0]), ["bystander==1.5"]);
// Phase B, alpha: badone at its planned version, and the bystander.
assert_eq!(rendered(&constraints[1]), ["badone==1.0", "bystander==1.5"]);
// Phase B, badone: alpha at its planned version, and the bystander.
assert_eq!(rendered(&constraints[2]), ["alpha==2.0", "bystander==1.5"]);
}
#[tokio::test]
async fn an_uncancelled_run_does_not_claim_it_was_cancelled() {
let summary = run(&FakeEngine::new(&[]), &[("alpha", "1.0")]).await;