feat(flow): PipDock enforces Requires-Python itself, as hard invariant 10 says

Invariant 10 says PipDock enforces Requires-Python because the engines disagree, and nothing
did (C2): PD-PKG-001 existed in the catalog and was never raised. pip refused an incompatible
exact pin with the "No matching distribution" it gives a typo (PD-PKG-002); uv honours the
lower bound, ignores the upper one, planned scipy==1.7.3 on Python 3.14 and failed at install
time building it from source.

The update flow now checks three places:
- exact pins the user typed (name==v, name===v), before any engine sees them. A request that is
  nothing but such pins is PD-PKG-001 with the range and the interpreter; otherwise the pin is
  left out and the preview says so (report.incompatible).
- forced pins: forcing a release this Python cannot run stays held back, with the reason.
- what the engine planned. pip's report already says what each release requires, so nothing is
  asked; for uv's plan PyPI is, and each release that excludes this Python is ruled out by name
  (`name!=v`) and the engine asked again, at most twice - it, not PipDock, picks what is next.

crate::release asks https://pypi.org/pypi/{name}/{version}/json for info.requires_python, at most
eight at once with a 15 s watchdog each, and remembers every answer forever in a new store table
(an uploaded release's metadata never changes). A failure is asked again; unknown never blocks.
The PyPI client is shared per process now, rather than rebuilt for every request (M37, part).

Store v4. Migrations ran on every open - a journal_mode write and eight CREATE TABLEs, on a store
some paths open twice per lookup - and an older build set user_version back down, so the next
newer build re-ran them (M38). Both fixed, with tests.

Verified on Python 3.14: `install scipy==1.7.3` is PD-PKG-001 under pip and uv alike, and
`install scipy==1.7.3 six==1.17.0` plans six and says why scipy was left out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 committed 2026-10-17 00:55:27 +07:00
1 parent 6ff2a10631
commit e2e75f63fb
13 files changed
+778 -21

No files matched your search

+14
View File
@@ -58,6 +58,9 @@ fn flow_options() -> FlowOptions {
allow_externally_managed: Store::open(&app_data_dir())
.and_then(|store| settings::load(&store))
.is_ok_and(|s| s.allow_externally_managed),
releases: Some(std::sync::Arc::new(
pipdock_core::release::PyPiReleases::new(app_data_dir()),
)),
..FlowOptions::default()
}
}
@@ -772,6 +775,17 @@ fn print_preview(opts: &GlobalOpts, report: &plan::ResolutionReport) {
}
}
}
if !report.incompatible.is_empty() {
// Hard invariant 10: left out, and why, rather than a "not found" for a version that
// exists.
println!("\nLeft out — needs a different Python:");
for m in &report.incompatible {
println!(
" {} {} requires Python {} (this is {})",
m.pkg, m.version, m.requires_python, m.python
);
}
}
if !report.forced.is_empty() {
// DISCLAIMER §2: what forcing breaks is said before anything runs, not discovered after.
println!("\nForced — will break:");
+10
View File
@@ -104,6 +104,7 @@ pub fn pip_report(stdout: &str, stderr: &str, installed: &[Dist]) -> Result<Pars
let mut changes = Vec::new();
let mut yanked = Vec::new();
let mut declared_python = std::collections::BTreeMap::new();
for entry in doc
.get("install")
@@ -128,6 +129,13 @@ pub fn pip_report(stdout: &str, stderr: &str, installed: &[Dist]) -> Result<Pars
continue;
}
let to = Version(raw_version.to_owned());
if let Some(requires) = meta
.get("requires_python")
.and_then(serde_json::Value::as_str)
.filter(|s| !s.trim().is_empty())
{
declared_python.insert(name.clone(), requires.to_owned());
}
if entry.get("is_yanked").and_then(serde_json::Value::as_bool) == Some(true) {
yanked.push(YankWarning {
@@ -166,6 +174,7 @@ pub fn pip_report(stdout: &str, stderr: &str, installed: &[Dist]) -> Result<Pars
// pip's stderr — its warnings — not the `--report` document on stdout, which carries
// every package's README and is already parsed above.
raw: stderr.to_owned(),
declared_python,
},
yanked,
})
@@ -436,6 +445,7 @@ pub fn uv_plan(stdout: &str, stderr: &str, installed: &[Dist]) -> Result<ParsedP
forced: Vec::new(),
yanked: Vec::new(),
raw: text.to_owned(),
declared_python: Default::default(),
},
yanked: uv_yank_warnings(text),
})
+1
View File
@@ -154,6 +154,7 @@ fn flow_step() -> crate::flow::FlowStep {
forced: Vec::new(),
yanked: Vec::new(),
raw: String::new(),
declared_python: Default::default(),
},
round: 0,
rounds_remaining: 3,
+187 -7
View File
@@ -35,13 +35,15 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::sync::Arc;
use crate::engine::{Engine, EventSink, ProgressSink, SystemPackages};
use crate::errors::{Code, PdError, Result};
use crate::graph::ReverseDeps;
use crate::model::{Dist, OutdatedDist, PkgName, PyEnv, Spec};
use crate::model::{Dist, EngineId, OutdatedDist, PinnedSpec, PkgName, PyEnv, Spec, Version};
use crate::pins::{self, Pin};
use crate::plan::{self, Decision, PlanRequest, ResolutionReport, Strategy};
use crate::plan::{self, Decision, PlanRequest, PythonMismatch, ResolutionReport, Strategy};
use crate::release::{ReleaseMeta, mismatch, requires_python_of};
use crate::snapshot::{self, Snapshot, SnapshotProof};
use tokio_util::sync::CancellationToken;
@@ -154,6 +156,10 @@ pub struct FlowOptions {
/// environment regardless (H11). On its own it still changes nothing — the flag is passed only
/// when this is on **and** the probe finds the marker.
pub allow_externally_managed: bool,
/// Where `Requires-Python` of a release comes from when the engine does not say — uv's plan,
/// and any exact pin before it reaches an engine (hard invariant 10). `None` checks only what
/// the engine itself reported.
pub releases: Option<Arc<dyn ReleaseMeta>>,
}
impl Default for FlowOptions {
@@ -162,6 +168,7 @@ impl Default for FlowOptions {
cancel: CancellationToken::new(),
clock: jiff::Timestamp::now,
allow_externally_managed: false,
releases: None,
}
}
}
@@ -237,6 +244,11 @@ pub struct UpdateFlow {
excluded_pins: Vec<Pin>,
/// Whether the PEP 668 override applies to this flow's engine calls.
system: SystemPackages,
/// See [`FlowOptions::releases`].
releases: Option<Arc<dyn ReleaseMeta>>,
/// Pins left out because this environment's Python cannot install them — every report this
/// flow produces says so (`PD-PKG-001`).
left_out: Vec<PythonMismatch>,
cancel: CancellationToken,
clock: fn() -> jiff::Timestamp,
}
@@ -290,11 +302,18 @@ impl UpdateFlow {
snapshot: SnapshotState::NotTaken,
excluded_pins,
system,
releases: opts.releases,
left_out: Vec::new(),
cancel: opts.cancel,
clock: opts.clock,
};
flow.screen_exact_pins().await;
if flow.req.upgrades.is_empty() && flow.req.installs.is_empty() {
// Everything asked for needs a different Python: say that, not "nothing to do".
if let Some(m) = flow.left_out.first() {
return Err(python_refusal(m));
}
return Ok((
flow,
FlowStep::Nothing {
@@ -339,10 +358,11 @@ impl UpdateFlow {
// read it, so forcing re-resolved exactly the preview the user had just been shown (C6).
// The forced package is pinned to its latest and the packages holding it back leave the
// guard set — which is what makes them the "Will break" rows.
let forced: Vec<PkgName> = match &self.req.strategy {
let mut forced: Vec<PkgName> = match &self.req.strategy {
Strategy::ForceLatest { overrides } => overrides.clone(),
Strategy::Compatible => Vec::new(),
};
self.screen_forced(&mut forced).await;
let mut adjust = if forced.is_empty() {
crate::engine::GuardAdjust::default()
} else {
@@ -355,6 +375,9 @@ impl UpdateFlow {
};
let mut rounds = 0;
let mut python_rounds = 0;
// Releases ruled out for their Python, and any still planned once the rounds ran out.
let mut ruled_out: Vec<PythonMismatch> = Vec::new();
let mut report = loop {
let ctx = crate::engine::ResolveCtx {
cancel: self.cancel.clone(),
@@ -372,14 +395,47 @@ impl UpdateFlow {
Err(e) => return Err(e),
};
let collisions = plan::forced_collisions(&report, &adjust, &self.req, &self.installed);
if collisions.is_empty() || rounds >= plan::FORCE_RELEASE_ROUNDS {
break report;
if !collisions.is_empty() && rounds < plan::FORCE_RELEASE_ROUNDS {
adjust.release.extend(collisions);
rounds += 1;
continue;
}
adjust.release.extend(collisions);
rounds += 1;
// Hard invariant 10 for what the engine chose: uv ignores an upper bound on
// `Requires-Python` and plans a release this interpreter cannot run. Each one is ruled
// out by name and the engine asked again — it, not PipDock, picks what comes next.
let violations = if report.impossible.is_none() {
self.screen_plan(&report).await
} else {
Vec::new()
};
if !violations.is_empty() && python_rounds < plan::PYTHON_EXCLUDE_ROUNDS {
adjust.exclude.extend(violations.iter().map(|m| PinnedSpec {
name: m.pkg.clone(),
version: m.version.clone(),
}));
ruled_out.extend(violations);
python_rounds += 1;
continue;
}
ruled_out.extend(violations);
break report;
};
plan::derive_held_back(&mut report, &self.req.upgrades, &self.outdated, &self.graph);
// A release ruled out for its Python is not held back by anything installed. The
// incompatible row says why; a held-back row would blame nobody.
report.held_back.retain(|h| {
!ruled_out
.iter()
.any(|m| m.pkg == h.pkg && m.version == h.latest)
});
plan::derive_forced(&mut report, &forced, &self.outdated, &self.graph);
let mut incompatible = self.left_out.clone();
for m in ruled_out {
if !incompatible.contains(&m) {
incompatible.push(m);
}
}
report.incompatible = incompatible;
self.report = report;
self.resolved_at = (self.clock)();
@@ -525,6 +581,130 @@ impl UpdateFlow {
pub fn cancel(&self) {
self.cancel.cancel();
}
/// Hard invariant 10 for the pins the user typed: an exact `name==v` this environment's
/// Python cannot install is left out before an engine sees it, and the preview says so.
///
/// Neither engine said it. pip refused such a pin with the "No matching distribution" it
/// gives a typo (PD-PKG-002), and uv planned it and failed at install time, building from
/// source. PD-PKG-001 existed in the catalog and nothing raised it (C2).
async fn screen_exact_pins(&mut self) {
let Some(meta) = &self.releases else {
return;
};
let exact: Vec<(PkgName, Version)> =
self.req.installs.iter().filter_map(exact_pin).collect();
if exact.is_empty() {
return;
}
let known = requires_python_of(meta, exact.clone()).await;
let bad: Vec<PythonMismatch> = exact
.iter()
.filter_map(|(name, version)| {
let declared = known.get(&(name.clone(), version.0.clone()));
mismatch(
name,
version,
declared.map(String::as_str),
&self.env.python_version,
)
})
.collect();
self.req
.installs
.retain(|s| !bad.iter().any(|m| m.pkg == s.name));
self.left_out.extend(bad);
}
/// The same for a forced package: forcing a release this Python cannot run is not a choice
/// worth acting on, so it stays where it was and the preview says why.
async fn screen_forced(&mut self, forced: &mut Vec<PkgName>) {
let Some(meta) = &self.releases else {
return;
};
let latest: Vec<(PkgName, Version)> = forced
.iter()
.filter_map(|f| {
self.outdated
.iter()
.find(|o| &o.name == f)
.map(|o| (o.name.clone(), o.latest.clone()))
})
.collect();
if latest.is_empty() {
return;
}
let known = requires_python_of(meta, latest.clone()).await;
for (name, version) in latest {
let declared = known.get(&(name.clone(), version.0.clone()));
if let Some(m) = mismatch(
&name,
&version,
declared.map(String::as_str),
&self.env.python_version,
) {
forced.retain(|f| f != &name);
if !self.left_out.contains(&m) {
self.left_out.push(m);
}
}
}
}
/// Every planned release whose `Requires-Python` excludes this environment.
///
/// pip's report says what each release requires, so nothing is asked; uv's plan says nothing,
/// so PyPI is (`crate::release`), and what it cannot answer does not block.
async fn screen_plan(&self, report: &ResolutionReport) -> Vec<PythonMismatch> {
let asked = match (&self.releases, self.engine.id()) {
(Some(meta), EngineId::Uv) => {
let unknown: Vec<(PkgName, Version)> = report
.changes
.iter()
.filter(|c| !report.declared_python.contains_key(&c.name))
.map(|c| (c.name.clone(), c.to.clone()))
.collect();
requires_python_of(meta, unknown).await
}
_ => BTreeMap::new(),
};
report
.changes
.iter()
.filter_map(|c| {
let declared = report
.declared_python
.get(&c.name)
.or_else(|| asked.get(&(c.name.clone(), c.to.0.clone())));
mismatch(
&c.name,
&c.to,
declared.map(String::as_str),
&self.env.python_version,
)
})
.collect()
}
}
/// `name==v` or `name===v` with a single exact version — the pins worth checking before the
/// engine sees them. A range or a wildcard is the engine's to choose within.
fn exact_pin(spec: &Spec) -> Option<(PkgName, Version)> {
let req = spec.version_req.as_deref()?;
let version = req.strip_prefix("===").or_else(|| req.strip_prefix("=="))?;
(crate::pep440::is_version(version) && !version.contains('*'))
.then(|| (spec.name.clone(), Version(version.to_owned())))
}
/// `PD-PKG-001` for a request that is nothing but pins this Python cannot install.
fn python_refusal(m: &PythonMismatch) -> PdError {
PdError::new(
Code::PkgRequiresPython,
format!(
"{} {} requires Python {}; this environment has {}",
m.pkg, m.version, m.requires_python, m.python
),
)
}
/// The uninstall flow (DATA-FLOW §5).
+14
View File
@@ -532,6 +532,20 @@ async fn fetch_metadata(name: &PkgName) -> Result<PackageMeta> {
})
}
/// One PyPI client for the process, for the many small requests a plan's lookups make — each
/// [`http_client`] call builds a connection pool and a TLS configuration from scratch.
///
/// # Errors
/// `PD-INT-001` when the client cannot be built.
pub(crate) fn shared_client() -> Result<&'static reqwest::Client> {
static CLIENT: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();
if let Some(client) = CLIENT.get() {
return Ok(client);
}
let client = http_client()?;
Ok(CLIENT.get_or_init(|| client))
}
/// An HTTPS client restricted to PyPI's hosts.
///
/// SECURITY §4: certificate verification is always on and redirects may not leave `pypi.org` or
+1
View File
@@ -40,6 +40,7 @@ pub mod model;
pub mod pep440;
pub mod pins;
pub mod plan;
pub mod release;
pub mod report;
pub mod requirements;
pub mod settings;
+11
View File
@@ -192,6 +192,11 @@ pub struct ResolutionReport {
#[serde(skip)]
#[schemars(skip)]
pub raw: String,
/// What the engine itself said each planned release requires of Python — pip's report carries
/// it, uv's plan does not. For the flow's own check (hard invariant 10); never on the wire.
#[serde(skip)]
#[schemars(skip)]
pub declared_python: std::collections::BTreeMap<PkgName, String>,
}
impl ResolutionReport {
@@ -397,6 +402,10 @@ pub const MAX_CONFLICT_ROUNDS: u8 = 3;
/// [`preview::forced_collisions`].
pub const FORCE_RELEASE_ROUNDS: usize = 3;
/// How many times one resolve may rule out releases whose `Requires-Python` excludes this
/// environment and ask again, before what remains is shown as a warning.
pub const PYTHON_EXCLUDE_ROUNDS: usize = 2;
/// A [`ResolutionReport`] the user has confirmed.
///
/// **This type is how DATA-FLOW §9.1 stops being a rule and starts being a fact.** There is no
@@ -1145,6 +1154,7 @@ mod tests {
forced: Vec::new(),
yanked: Vec::new(),
raw: String::new(),
declared_python: Default::default(),
};
assert!(!report.is_clean());
}
@@ -1164,6 +1174,7 @@ mod tests {
forced: Vec::new(),
yanked: Vec::new(),
raw: String::new(),
declared_python: Default::default(),
};
assert!(report.is_clean());
let pinned = report.pinned_set();
+1
View File
@@ -308,6 +308,7 @@ mod tests {
forced: Vec::new(),
yanked: Vec::new(),
raw: String::new(),
declared_python: Default::default(),
}
}
+235
View File
@@ -0,0 +1,235 @@
//! What a release requires of the interpreter — the half of PipDock's own `Requires-Python`
//! enforcement an engine's plan cannot always supply (hard invariant 10).
//!
//! The engines disagree, which is why PipDock checks at all: pip filters candidates by
//! `Requires-Python` and refuses an incompatible exact pin with the same "No matching
//! distribution" it gives a typo; uv honours the lower bound and **ignores an upper bound**, so it
//! plans `scipy==1.7.3` (`<3.10`) on Python 3.14 and the install fails later, building from
//! source. pip's report carries each planned release's `requires_python`; uv's plan is text and
//! carries nothing, so for uv — and for an exact pin under either engine, checked before the
//! engine runs — the answer comes from PyPI's per-release JSON.
//!
//! **Unknown never blocks.** A release PyPI does not know (a private index), a network that is
//! down, a specifier that will not parse: each is "do not block", because refusing an installable
//! package on PipDock's uncertainty is the worse failure, and the engine still judges.
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use crate::model::{PkgName, Version};
use crate::store::Store;
/// How many lookups may be in flight at once. A plan for "update everything" can name fifty
/// releases; PyPI is a CDN, but fifty simultaneous connections from one desktop app is rude.
pub const LOOKUP_CONCURRENCY: usize = 8;
/// Per-request watchdog. A lookup that takes longer is treated as unknown, which never blocks.
pub const LOOKUP_TIMEOUT: Duration = Duration::from_secs(15);
/// Where `Requires-Python` of a release comes from.
#[async_trait]
pub trait ReleaseMeta: Send + Sync + std::fmt::Debug {
/// `Requires-Python` of `name==version`; `None` when it declares none **or nobody knows** —
/// both mean "do not block".
async fn requires_python(&self, name: &PkgName, version: &Version) -> Option<String>;
}
/// PyPI's per-release JSON, remembered in the store's `release_meta` table **forever**: an
/// uploaded release's metadata never changes, so a row is never refreshed. Only an answer is
/// remembered; a failure is asked again next time.
#[derive(Debug, Clone)]
pub struct PyPiReleases {
app_data: PathBuf,
}
impl PyPiReleases {
/// Lookups cached under `app_data`'s store.
#[must_use]
pub fn new(app_data: impl Into<PathBuf>) -> Self {
Self {
app_data: app_data.into(),
}
}
/// `Some(answer)` when this release has been asked about before.
///
/// The store is opened for this one read and dropped: `Store` is not `Sync`, and holding it
/// across the network call would make every caller's future not `Send`.
fn cached(&self, name: &PkgName, version: &Version) -> Option<Option<String>> {
let store = Store::open(&self.app_data).ok()?;
store
.conn()
.query_row(
"SELECT requires_python FROM release_meta WHERE normalized = ?1 AND version = ?2",
rusqlite::params![name.as_str(), version.0],
|r| r.get::<_, Option<String>>(0),
)
.ok()
}
fn remember(&self, name: &PkgName, version: &Version, requires_python: Option<&str>) {
// A cache that cannot be written only costs the next run a request.
if let Ok(store) = Store::open(&self.app_data) {
let _ = store.conn().execute(
"INSERT INTO release_meta (normalized, version, requires_python)
VALUES (?1, ?2, ?3)
ON CONFLICT (normalized, version) DO UPDATE SET
requires_python = excluded.requires_python",
rusqlite::params![name.as_str(), version.0, requires_python],
);
}
}
}
#[async_trait]
impl ReleaseMeta for PyPiReleases {
async fn requires_python(&self, name: &PkgName, version: &Version) -> Option<String> {
if let Some(answer) = self.cached(name, version) {
return answer;
}
let answer = fetch(name, version).await?;
self.remember(name, version, answer.as_deref());
answer
}
}
/// `GET https://pypi.org/pypi/{name}/{version}/json` → `info.requires_python`.
///
/// `Some(None)` is a real answer — the release declares no constraint. `None` is "could not
/// find out": not on PyPI, unreachable, slow, unreadable.
async fn fetch(name: &PkgName, version: &Version) -> Option<Option<String>> {
// The version goes into a URL path; only a PEP 440 version is allowed to.
if !crate::pep440::is_version(&version.0) {
return None;
}
let url = format!("https://pypi.org/pypi/{}/{}/json", name.as_str(), version.0);
let response = crate::index::shared_client()
.ok()?
.get(&url)
.timeout(LOOKUP_TIMEOUT)
.send()
.await
.ok()?;
if !response.status().is_success() {
return None;
}
let doc: serde_json::Value = response.json().await.ok()?;
Some(
doc.get("info")
.and_then(|i| i.get("requires_python"))
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_owned),
)
}
/// Look up many releases at once, at most [`LOOKUP_CONCURRENCY`] in flight. Releases with no
/// known constraint are absent from the answer.
pub async fn requires_python_of(
meta: &Arc<dyn ReleaseMeta>,
releases: Vec<(PkgName, Version)>,
) -> BTreeMap<(PkgName, String), String> {
let gate = Arc::new(tokio::sync::Semaphore::new(LOOKUP_CONCURRENCY));
let mut set = tokio::task::JoinSet::new();
for (name, version) in releases {
let meta = Arc::clone(meta);
let gate = Arc::clone(&gate);
set.spawn(async move {
let _permit = gate.acquire_owned().await.ok();
let answer = meta.requires_python(&name, &version).await;
(name, version, answer)
});
}
let mut out = BTreeMap::new();
while let Some(joined) = set.join_next().await {
if let Ok((name, version, Some(requires))) = joined {
out.insert((name, version.0), requires);
}
}
out
}
/// `Some` when `requires_python` excludes the interpreter at `python` (e.g. `"3.14.7"`).
///
/// An unparseable interpreter version or specifier is not a mismatch — see the module note.
#[must_use]
pub fn mismatch(
pkg: &PkgName,
version: &Version,
requires_python: Option<&str>,
python: &str,
) -> Option<crate::plan::PythonMismatch> {
let interpreter = crate::compat::PyVersion::parse(python).ok()?;
match crate::compat::check(requires_python, &interpreter) {
crate::compat::Compatibility::RequiresPython { required, .. } => {
Some(crate::plan::PythonMismatch {
pkg: pkg.clone(),
version: version.clone(),
requires_python: required,
python: python.to_owned(),
})
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn pkg(n: &str) -> PkgName {
PkgName::parse(n).unwrap()
}
#[test]
fn a_capped_release_on_a_newer_python_is_a_mismatch() {
let m = mismatch(
&pkg("scipy"),
&Version("1.7.3".into()),
Some(">=3.7,<3.10"),
"3.14.7",
)
.expect("excluded");
assert_eq!(m.requires_python, ">=3.7,<3.10");
assert_eq!(m.python, "3.14.7");
assert!(
mismatch(
&pkg("scipy"),
&Version("1.16.0".into()),
Some(">=3.11"),
"3.14.7"
)
.is_none()
);
}
#[test]
fn what_cannot_be_read_never_blocks() {
let v = Version("1.0".into());
assert!(mismatch(&pkg("a"), &v, None, "3.14.7").is_none());
assert!(mismatch(&pkg("a"), &v, Some(""), "3.14.7").is_none());
assert!(mismatch(&pkg("a"), &v, Some("not a specifier"), "3.14.7").is_none());
assert!(mismatch(&pkg("a"), &v, Some("<3.10"), "not a version").is_none());
}
#[test]
fn an_answer_is_remembered_and_a_failure_is_not() {
let dir = std::env::temp_dir().join(format!("pd-release-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let meta = PyPiReleases::new(&dir);
let (name, v) = (pkg("scipy"), Version("1.7.3".into()));
assert_eq!(meta.cached(&name, &v), None, "never asked");
meta.remember(&name, &v, Some(">=3.7,<3.10"));
assert_eq!(meta.cached(&name, &v), Some(Some(">=3.7,<3.10".to_owned())));
// "Declares nothing" is an answer too, and is not asked again.
let free = Version("1.0".into());
meta.remember(&name, &free, None);
assert_eq!(meta.cached(&name, &free), Some(None));
let _ = std::fs::remove_dir_all(&dir);
}
}
+66 -5
View File
@@ -5,9 +5,10 @@
//! lifetime — deleting the app data folder is documented as a complete reset (SECURITY §8), and
//! that promise is easier to keep with one file than four.
//!
//! Migrations are forward-only and idempotent, applied on every open. There is no downgrade path:
//! an older PipDock opening a newer database is a situation the updater does not create, and
//! pretending to support it would mean writing migrations nobody can test.
//! Migrations are forward-only and idempotent, and run only when the database's `user_version` is
//! older than this build's. There is no downgrade path, but an older PipDock opening a newer
//! database must not *lower* the version either — it used to set it unconditionally, so the next
//! newer build re-ran every migration.
use std::path::{Path, PathBuf};
@@ -19,7 +20,7 @@ use crate::errors::{Code, PdError, Result};
pub const DB_FILE: &str = "index.db";
/// Schema version this build expects. Bump when adding a migration.
pub const SCHEMA_VERSION: i64 = 3;
pub const SCHEMA_VERSION: i64 = 4;
/// An open store.
#[derive(Debug)]
@@ -117,10 +118,23 @@ impl Store {
PdError::new(Code::IntUnexpected, format!("store migration failed: {e}"))
};
// Per connection, so every open; everything below is per database, and the database
// remembers it. Every open used to run the whole batch — a `journal_mode` write and eight
// `CREATE TABLE`s — on a store some paths open twice per metadata lookup (M38).
self.conn
.execute_batch("PRAGMA foreign_keys = ON;")
.map_err(sql)?;
let current: i64 = self
.conn
.pragma_query_value(None, "user_version", |r| r.get(0))
.map_err(sql)?;
if current >= SCHEMA_VERSION {
return Ok(());
}
self.conn
.execute_batch(
"PRAGMA journal_mode = WAL;
PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS names (
name TEXT NOT NULL,
@@ -168,6 +182,16 @@ impl Store {
CREATE TABLE IF NOT EXISTS kv (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
-- v4. `Requires-Python` of one release, as PyPI publishes it (crate::release).
-- An uploaded release's metadata never changes, so a row is never refreshed;
-- NULL is the answer \"declares none\", not \"unknown\", which is never stored.
CREATE TABLE IF NOT EXISTS release_meta (
normalized TEXT NOT NULL,
version TEXT NOT NULL,
requires_python TEXT,
PRIMARY KEY (normalized, version)
);",
)
.map_err(sql)?;
@@ -397,4 +421,41 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_older_build_never_lowers_the_version() {
let dir = std::env::temp_dir().join(format!("pd-store-newer-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let store = Store::open(&dir).expect("open");
store
.conn()
.pragma_update(None, "user_version", SCHEMA_VERSION + 1)
.expect("a newer build was here");
drop(store);
let reopened = Store::open(&dir).expect("this build opens it");
let version: i64 = reopened
.conn()
.pragma_query_value(None, "user_version", |r| r.get(0))
.expect("version");
assert_eq!(version, SCHEMA_VERSION + 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_v3_store_gains_the_release_cache() {
let store = Store::in_memory().expect("open");
store
.conn()
.execute_batch("DROP TABLE release_meta; PRAGMA user_version = 3;")
.expect("rewind to v3");
store.migrate().expect("migrate");
store
.conn()
.execute(
"INSERT INTO release_meta (normalized, version, requires_python) VALUES ('a', '1', NULL)",
[],
)
.expect("the v4 table exists");
}
}
+2
View File
@@ -125,6 +125,7 @@ impl Engine for FakeEngine {
forced: Vec::new(),
yanked: Vec::new(),
raw: String::new(),
declared_python: Default::default(),
})
}
async fn install(
@@ -291,6 +292,7 @@ fn report(packages: &[(&str, &str)]) -> ResolutionReport {
forced: Vec::new(),
yanked: Vec::new(),
raw: String::new(),
declared_python: Default::default(),
}
}
+233 -9
View File
@@ -47,6 +47,11 @@ struct World {
installed: BTreeMap<String, Installed>,
/// The newest release the index offers, per package.
latest: BTreeMap<String, String>,
/// Older releases the index also offers, newest first — what a resolver falls back to when
/// the latest is ruled out.
older: BTreeMap<String, Vec<String>>,
/// Which engine this pretends to be; the flow asks PyPI about uv's plans, not pip's.
engine: Option<EngineId>,
/// What a package's newest release needs at least, per dependency: `httpcore` 1.0.9 needs
/// `h11>=0.16`.
needs: BTreeMap<String, Vec<(String, String)>>,
@@ -63,10 +68,6 @@ struct World {
/// What one resolve was asked: the request, what the flow changed about the guard set, and
/// whether the PEP 668 override applied.
#[expect(
dead_code,
reason = "recorded whole, for whichever part a test asserts on"
)]
struct ResolveCall {
req: PlanRequest,
adjust: GuardAdjust,
@@ -101,6 +102,19 @@ impl Scripted {
self
}
fn also_offering(self, name: &str, older: &[&str]) -> Self {
self.world().older.insert(
name.to_owned(),
older.iter().map(|v| (*v).to_owned()).collect(),
);
self
}
fn pretending_to_be(self, id: EngineId) -> Self {
self.world().engine = Some(id);
self
}
fn needing(self, name: &str, dep: &str, at_least: &str) -> Self {
self.world()
.needs
@@ -152,7 +166,7 @@ fn env() -> PyEnv {
#[async_trait]
impl Engine for Scripted {
fn id(&self) -> EngineId {
EngineId::Pip
self.world().engine.unwrap_or(EngineId::Pip)
}
async fn info(&self, _env: &PyEnv) -> EngineInfo {
@@ -238,15 +252,41 @@ impl Engine for Scripted {
}
}
let excluded = |name: &str, version: &str| {
ctx.adjust
.exclude
.iter()
.any(|e| e.name.as_str() == name && e.version.0 == version)
};
let mut changes = Vec::new();
for spec in &req.installs {
let Some(version) = spec
.version_req
.as_deref()
.and_then(|v| v.strip_prefix("=="))
else {
continue;
};
changes.push(Change {
name: spec.name.clone(),
from: None,
to: Version(version.to_owned()),
kind: ChangeKind::NewInstall,
});
}
for name in &req.upgrades {
if pinned.contains(&name.as_str()) {
continue;
}
let (Some(from), Some(to)) = (
world.installed.get(name.as_str()),
world.latest.get(name.as_str()),
) else {
let Some(from) = world.installed.get(name.as_str()) else {
continue;
};
let candidates = world
.latest
.get(name.as_str())
.into_iter()
.chain(world.older.get(name.as_str()).into_iter().flatten());
let Some(to) = candidates.into_iter().find(|v| !excluded(name.as_str(), v)) else {
continue;
};
let capped = guard
@@ -861,3 +901,187 @@ async fn force_latest_forces_and_names_what_it_breaks() {
assert!(last.release.contains(&pkg("httpx")));
assert!(last.release.contains(&pkg("h11")));
}
/// `Requires-Python` per release, as PyPI would answer; anything absent is unknown.
#[derive(Debug, Default)]
struct Releases {
known: BTreeMap<(String, String), String>,
asked: std::sync::atomic::AtomicUsize,
}
impl Releases {
fn with(mut self, name: &str, version: &str, requires: &str) -> Self {
self.known
.insert((name.to_owned(), version.to_owned()), requires.to_owned());
self
}
}
#[async_trait]
impl pipdock_core::release::ReleaseMeta for Releases {
async fn requires_python(&self, name: &PkgName, version: &Version) -> Option<String> {
self.asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
self.known
.get(&(name.to_string(), version.0.clone()))
.cloned()
}
}
fn checking(releases: &Arc<Releases>) -> FlowOptions {
FlowOptions {
releases: Some(Arc::clone(releases) as Arc<dyn pipdock_core::release::ReleaseMeta>),
..FlowOptions::default()
}
}
fn install(specs: &[&str]) -> Intent {
Intent::Install {
specs: specs.iter().map(|s| (*s).to_owned()).collect(),
}
}
#[tokio::test]
async fn an_exact_pin_this_python_cannot_install_is_pd_pkg_001_before_any_engine_runs() {
// C2. pip answered this with the "No matching distribution" it gives a typo, and uv planned
// it and failed at install time, building scipy from source. PD-PKG-001 existed in the
// catalog and nothing ever raised it.
let releases = Arc::new(Releases::default().with("scipy", "1.7.3", ">=3.7,<3.10"));
for engine_id in [EngineId::Pip, EngineId::Uv] {
let engine = Scripted::default().pretending_to_be(engine_id);
let Err(err) = UpdateFlow::start(
env(),
Box::new(engine.clone()),
&install(&["scipy==1.7.3"]),
&[],
checking(&releases),
)
.await
else {
panic!("{engine_id:?}: an install this Python cannot run must be refused");
};
assert_eq!(err.code, Code::PkgRequiresPython, "{engine_id:?}");
assert!(err.message.contains(">=3.7,<3.10") && err.message.contains("3.12.4"));
assert!(
engine.world().resolves.is_empty(),
"refused before the engine ran"
);
}
}
#[tokio::test]
async fn one_pin_this_python_cannot_install_does_not_cost_the_others() {
let releases = Arc::new(Releases::default().with("scipy", "1.7.3", ">=3.7,<3.10"));
let engine = Scripted::default();
let (_, step) = UpdateFlow::start(
env(),
Box::new(engine.clone()),
&install(&["scipy==1.7.3", "idna==3.10"]),
&[],
checking(&releases),
)
.await
.expect("start");
let FlowStep::NeedsConfirm { report } = step else {
panic!("the rest of the request is confirmable: {step:?}");
};
let planned: Vec<String> = report.changes.iter().map(|c| c.name.to_string()).collect();
assert_eq!(planned, ["idna"]);
assert_eq!(report.incompatible.len(), 1);
assert_eq!(report.incompatible[0].pkg.as_str(), "scipy");
assert_eq!(report.incompatible[0].requires_python, ">=3.7,<3.10");
let world = engine.world();
let asked: Vec<String> = world.resolves[0]
.req
.installs
.iter()
.map(|s| s.name.to_string())
.collect();
assert_eq!(asked, ["idna"], "scipy never reached the engine");
}
#[tokio::test]
async fn a_release_uv_planned_past_its_python_cap_is_ruled_out_and_the_engine_asked_again() {
// uv honours a lower Requires-Python bound and ignores an upper one, so it plans numba
// 0.62.0 (<3.14) on 3.14. PipDock rules that release out by name and uv picks again.
let releases = Arc::new(
Releases::default()
.with("numba", "0.62.0", ">=3.10,<3.14")
.with("numba", "0.61.2", ">=3.10,<3.15"),
);
let engine = Scripted::default()
.pretending_to_be(EngineId::Uv)
.with("numba", "0.60.0", &[])
.offering("numba", "0.62.0")
.also_offering("numba", &["0.61.2"]);
let mut py314 = env();
py314.python_version = "3.14.7".into();
let (_, step) = UpdateFlow::start(
py314,
Box::new(engine.clone()),
&update_all(),
&[],
checking(&releases),
)
.await
.expect("start");
let FlowStep::NeedsConfirm { report } = step else {
panic!("an older compatible release is a clean plan, not a held-back row: {step:?}");
};
assert_eq!(report.changes.len(), 1);
assert_eq!(report.changes[0].to.0, "0.61.2");
assert_eq!(
report.incompatible.len(),
1,
"and the preview says what was ruled out"
);
assert_eq!(report.incompatible[0].version.0, "0.62.0");
let world = engine.world();
assert_eq!(world.resolves.len(), 2);
assert_eq!(world.resolves[1].adjust.exclude[0].version.0, "0.62.0");
}
#[tokio::test]
async fn pip_s_own_answer_is_used_and_pypi_is_not_asked_about_its_plan() {
let releases = Arc::new(Releases::default());
let engine = Scripted::default()
.with("idna", "3.4", &[])
.offering("idna", "3.10");
let (_, step) = UpdateFlow::start(
env(),
Box::new(engine.clone()),
&update_all(),
&[],
checking(&releases),
)
.await
.expect("start");
assert!(matches!(step, FlowStep::NeedsConfirm { .. }), "{step:?}");
assert_eq!(
releases.asked.load(std::sync::atomic::Ordering::SeqCst),
0,
"nothing exact was pinned and pip's plan says what it requires"
);
}
#[tokio::test]
async fn a_requires_python_nobody_knows_never_blocks() {
// A private index, a network that is down: refusing an installable package on PipDock's
// uncertainty is the worse failure, and the engine still judges.
let releases = Arc::new(Releases::default());
let engine = Scripted::default().pretending_to_be(EngineId::Uv);
let (_, step) = UpdateFlow::start(
env(),
Box::new(engine),
&install(&["internal-tool==2.0"]),
&[],
checking(&releases),
)
.await
.expect("start");
let FlowStep::NeedsConfirm { report } = step else {
panic!("{step:?}");
};
assert_eq!(report.changes.len(), 1);
assert!(report.incompatible.is_empty());
assert!(releases.asked.load(std::sync::atomic::Ordering::SeqCst) >= 1);
}
+3
View File
@@ -665,6 +665,9 @@ pub async fn plan_resolve(
let opts = flow::FlowOptions {
cancel,
allow_externally_managed,
releases: Some(std::sync::Arc::new(
pipdock_core::release::PyPiReleases::new(state.app_data.clone()),
)),
..flow::FlowOptions::default()
};
match flow::UpdateFlow::start(env, engine, &intent, &env_pins, opts).await {