diff --git a/crates/pipdock-cli/src/run.rs b/crates/pipdock-cli/src/run.rs index 85a660c..6df52cb 100644 --- a/crates/pipdock-cli/src/run.rs +++ b/crates/pipdock-cli/src/run.rs @@ -58,6 +58,9 @@ fn flow_options() -> FlowOptions { allow_externally_managed: Store::open(&app_data_dir()) .and_then(|store| settings::load(&store)) .is_ok_and(|s| s.allow_externally_managed), + releases: Some(std::sync::Arc::new( + pipdock_core::release::PyPiReleases::new(app_data_dir()), + )), ..FlowOptions::default() } } @@ -772,6 +775,17 @@ fn print_preview(opts: &GlobalOpts, report: &plan::ResolutionReport) { } } } + if !report.incompatible.is_empty() { + // Hard invariant 10: left out, and why, rather than a "not found" for a version that + // exists. + println!("\nLeft out — needs a different Python:"); + for m in &report.incompatible { + println!( + " {} {} requires Python {} (this is {})", + m.pkg, m.version, m.requires_python, m.python + ); + } + } if !report.forced.is_empty() { // DISCLAIMER §2: what forcing breaks is said before anything runs, not discovered after. println!("\nForced — will break:"); diff --git a/crates/pipdock-core/src/engine/parse.rs b/crates/pipdock-core/src/engine/parse.rs index 227f6e2..7eaea3f 100644 --- a/crates/pipdock-core/src/engine/parse.rs +++ b/crates/pipdock-core/src/engine/parse.rs @@ -104,6 +104,7 @@ pub fn pip_report(stdout: &str, stderr: &str, installed: &[Dist]) -> Result Result Result Result crate::flow::FlowStep { forced: Vec::new(), yanked: Vec::new(), raw: String::new(), + declared_python: Default::default(), }, round: 0, rounds_remaining: 3, diff --git a/crates/pipdock-core/src/flow/mod.rs b/crates/pipdock-core/src/flow/mod.rs index babc493..e6e3c24 100644 --- a/crates/pipdock-core/src/flow/mod.rs +++ b/crates/pipdock-core/src/flow/mod.rs @@ -35,13 +35,15 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; +use std::sync::Arc; use crate::engine::{Engine, EventSink, ProgressSink, SystemPackages}; use crate::errors::{Code, PdError, Result}; use crate::graph::ReverseDeps; -use crate::model::{Dist, OutdatedDist, PkgName, PyEnv, Spec}; +use crate::model::{Dist, EngineId, OutdatedDist, PinnedSpec, PkgName, PyEnv, Spec, Version}; use crate::pins::{self, Pin}; -use crate::plan::{self, Decision, PlanRequest, ResolutionReport, Strategy}; +use crate::plan::{self, Decision, PlanRequest, PythonMismatch, ResolutionReport, Strategy}; +use crate::release::{ReleaseMeta, mismatch, requires_python_of}; use crate::snapshot::{self, Snapshot, SnapshotProof}; use tokio_util::sync::CancellationToken; @@ -154,6 +156,10 @@ pub struct FlowOptions { /// environment regardless (H11). On its own it still changes nothing — the flag is passed only /// when this is on **and** the probe finds the marker. pub allow_externally_managed: bool, + /// Where `Requires-Python` of a release comes from when the engine does not say — uv's plan, + /// and any exact pin before it reaches an engine (hard invariant 10). `None` checks only what + /// the engine itself reported. + pub releases: Option>, } impl Default for FlowOptions { @@ -162,6 +168,7 @@ impl Default for FlowOptions { cancel: CancellationToken::new(), clock: jiff::Timestamp::now, allow_externally_managed: false, + releases: None, } } } @@ -237,6 +244,11 @@ pub struct UpdateFlow { excluded_pins: Vec, /// Whether the PEP 668 override applies to this flow's engine calls. system: SystemPackages, + /// See [`FlowOptions::releases`]. + releases: Option>, + /// Pins left out because this environment's Python cannot install them — every report this + /// flow produces says so (`PD-PKG-001`). + left_out: Vec, cancel: CancellationToken, clock: fn() -> jiff::Timestamp, } @@ -290,11 +302,18 @@ impl UpdateFlow { snapshot: SnapshotState::NotTaken, excluded_pins, system, + releases: opts.releases, + left_out: Vec::new(), cancel: opts.cancel, clock: opts.clock, }; + flow.screen_exact_pins().await; if flow.req.upgrades.is_empty() && flow.req.installs.is_empty() { + // Everything asked for needs a different Python: say that, not "nothing to do". + if let Some(m) = flow.left_out.first() { + return Err(python_refusal(m)); + } return Ok(( flow, FlowStep::Nothing { @@ -339,10 +358,11 @@ impl UpdateFlow { // read it, so forcing re-resolved exactly the preview the user had just been shown (C6). // The forced package is pinned to its latest and the packages holding it back leave the // guard set — which is what makes them the "Will break" rows. - let forced: Vec = match &self.req.strategy { + let mut forced: Vec = match &self.req.strategy { Strategy::ForceLatest { overrides } => overrides.clone(), Strategy::Compatible => Vec::new(), }; + self.screen_forced(&mut forced).await; let mut adjust = if forced.is_empty() { crate::engine::GuardAdjust::default() } else { @@ -355,6 +375,9 @@ impl UpdateFlow { }; let mut rounds = 0; + let mut python_rounds = 0; + // Releases ruled out for their Python, and any still planned once the rounds ran out. + let mut ruled_out: Vec = Vec::new(); let mut report = loop { let ctx = crate::engine::ResolveCtx { cancel: self.cancel.clone(), @@ -372,14 +395,47 @@ impl UpdateFlow { Err(e) => return Err(e), }; let collisions = plan::forced_collisions(&report, &adjust, &self.req, &self.installed); - if collisions.is_empty() || rounds >= plan::FORCE_RELEASE_ROUNDS { - break report; + if !collisions.is_empty() && rounds < plan::FORCE_RELEASE_ROUNDS { + adjust.release.extend(collisions); + rounds += 1; + continue; } - adjust.release.extend(collisions); - rounds += 1; + // Hard invariant 10 for what the engine chose: uv ignores an upper bound on + // `Requires-Python` and plans a release this interpreter cannot run. Each one is ruled + // out by name and the engine asked again — it, not PipDock, picks what comes next. + let violations = if report.impossible.is_none() { + self.screen_plan(&report).await + } else { + Vec::new() + }; + if !violations.is_empty() && python_rounds < plan::PYTHON_EXCLUDE_ROUNDS { + adjust.exclude.extend(violations.iter().map(|m| PinnedSpec { + name: m.pkg.clone(), + version: m.version.clone(), + })); + ruled_out.extend(violations); + python_rounds += 1; + continue; + } + ruled_out.extend(violations); + break report; }; plan::derive_held_back(&mut report, &self.req.upgrades, &self.outdated, &self.graph); + // A release ruled out for its Python is not held back by anything installed. The + // incompatible row says why; a held-back row would blame nobody. + report.held_back.retain(|h| { + !ruled_out + .iter() + .any(|m| m.pkg == h.pkg && m.version == h.latest) + }); plan::derive_forced(&mut report, &forced, &self.outdated, &self.graph); + let mut incompatible = self.left_out.clone(); + for m in ruled_out { + if !incompatible.contains(&m) { + incompatible.push(m); + } + } + report.incompatible = incompatible; self.report = report; self.resolved_at = (self.clock)(); @@ -525,6 +581,130 @@ impl UpdateFlow { pub fn cancel(&self) { self.cancel.cancel(); } + + /// Hard invariant 10 for the pins the user typed: an exact `name==v` this environment's + /// Python cannot install is left out before an engine sees it, and the preview says so. + /// + /// Neither engine said it. pip refused such a pin with the "No matching distribution" it + /// gives a typo (PD-PKG-002), and uv planned it and failed at install time, building from + /// source. PD-PKG-001 existed in the catalog and nothing raised it (C2). + async fn screen_exact_pins(&mut self) { + let Some(meta) = &self.releases else { + return; + }; + let exact: Vec<(PkgName, Version)> = + self.req.installs.iter().filter_map(exact_pin).collect(); + if exact.is_empty() { + return; + } + let known = requires_python_of(meta, exact.clone()).await; + let bad: Vec = exact + .iter() + .filter_map(|(name, version)| { + let declared = known.get(&(name.clone(), version.0.clone())); + mismatch( + name, + version, + declared.map(String::as_str), + &self.env.python_version, + ) + }) + .collect(); + self.req + .installs + .retain(|s| !bad.iter().any(|m| m.pkg == s.name)); + self.left_out.extend(bad); + } + + /// The same for a forced package: forcing a release this Python cannot run is not a choice + /// worth acting on, so it stays where it was and the preview says why. + async fn screen_forced(&mut self, forced: &mut Vec) { + let Some(meta) = &self.releases else { + return; + }; + let latest: Vec<(PkgName, Version)> = forced + .iter() + .filter_map(|f| { + self.outdated + .iter() + .find(|o| &o.name == f) + .map(|o| (o.name.clone(), o.latest.clone())) + }) + .collect(); + if latest.is_empty() { + return; + } + let known = requires_python_of(meta, latest.clone()).await; + for (name, version) in latest { + let declared = known.get(&(name.clone(), version.0.clone())); + if let Some(m) = mismatch( + &name, + &version, + declared.map(String::as_str), + &self.env.python_version, + ) { + forced.retain(|f| f != &name); + if !self.left_out.contains(&m) { + self.left_out.push(m); + } + } + } + } + + /// Every planned release whose `Requires-Python` excludes this environment. + /// + /// pip's report says what each release requires, so nothing is asked; uv's plan says nothing, + /// so PyPI is (`crate::release`), and what it cannot answer does not block. + async fn screen_plan(&self, report: &ResolutionReport) -> Vec { + let asked = match (&self.releases, self.engine.id()) { + (Some(meta), EngineId::Uv) => { + let unknown: Vec<(PkgName, Version)> = report + .changes + .iter() + .filter(|c| !report.declared_python.contains_key(&c.name)) + .map(|c| (c.name.clone(), c.to.clone())) + .collect(); + requires_python_of(meta, unknown).await + } + _ => BTreeMap::new(), + }; + report + .changes + .iter() + .filter_map(|c| { + let declared = report + .declared_python + .get(&c.name) + .or_else(|| asked.get(&(c.name.clone(), c.to.0.clone()))); + mismatch( + &c.name, + &c.to, + declared.map(String::as_str), + &self.env.python_version, + ) + }) + .collect() + } +} + +/// `name==v` or `name===v` with a single exact version — the pins worth checking before the +/// engine sees them. A range or a wildcard is the engine's to choose within. +fn exact_pin(spec: &Spec) -> Option<(PkgName, Version)> { + let req = spec.version_req.as_deref()?; + let version = req.strip_prefix("===").or_else(|| req.strip_prefix("=="))?; + (crate::pep440::is_version(version) && !version.contains('*')) + .then(|| (spec.name.clone(), Version(version.to_owned()))) +} + +/// `PD-PKG-001` for a request that is nothing but pins this Python cannot install. +fn python_refusal(m: &PythonMismatch) -> PdError { + PdError::new( + Code::PkgRequiresPython, + format!( + "{} {} requires Python {}; this environment has {}", + m.pkg, m.version, m.requires_python, m.python + ), + ) } /// The uninstall flow (DATA-FLOW §5). diff --git a/crates/pipdock-core/src/index/mod.rs b/crates/pipdock-core/src/index/mod.rs index adfb018..225367e 100644 --- a/crates/pipdock-core/src/index/mod.rs +++ b/crates/pipdock-core/src/index/mod.rs @@ -532,6 +532,20 @@ async fn fetch_metadata(name: &PkgName) -> Result { }) } +/// One PyPI client for the process, for the many small requests a plan's lookups make — each +/// [`http_client`] call builds a connection pool and a TLS configuration from scratch. +/// +/// # Errors +/// `PD-INT-001` when the client cannot be built. +pub(crate) fn shared_client() -> Result<&'static reqwest::Client> { + static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + if let Some(client) = CLIENT.get() { + return Ok(client); + } + let client = http_client()?; + Ok(CLIENT.get_or_init(|| client)) +} + /// An HTTPS client restricted to PyPI's hosts. /// /// SECURITY §4: certificate verification is always on and redirects may not leave `pypi.org` or diff --git a/crates/pipdock-core/src/lib.rs b/crates/pipdock-core/src/lib.rs index 2d0b943..98fb8b0 100644 --- a/crates/pipdock-core/src/lib.rs +++ b/crates/pipdock-core/src/lib.rs @@ -40,6 +40,7 @@ pub mod model; pub mod pep440; pub mod pins; pub mod plan; +pub mod release; pub mod report; pub mod requirements; pub mod settings; diff --git a/crates/pipdock-core/src/plan/mod.rs b/crates/pipdock-core/src/plan/mod.rs index 9009d0f..d4e0c90 100644 --- a/crates/pipdock-core/src/plan/mod.rs +++ b/crates/pipdock-core/src/plan/mod.rs @@ -192,6 +192,11 @@ pub struct ResolutionReport { #[serde(skip)] #[schemars(skip)] pub raw: String, + /// What the engine itself said each planned release requires of Python — pip's report carries + /// it, uv's plan does not. For the flow's own check (hard invariant 10); never on the wire. + #[serde(skip)] + #[schemars(skip)] + pub declared_python: std::collections::BTreeMap, } impl ResolutionReport { @@ -397,6 +402,10 @@ pub const MAX_CONFLICT_ROUNDS: u8 = 3; /// [`preview::forced_collisions`]. pub const FORCE_RELEASE_ROUNDS: usize = 3; +/// How many times one resolve may rule out releases whose `Requires-Python` excludes this +/// environment and ask again, before what remains is shown as a warning. +pub const PYTHON_EXCLUDE_ROUNDS: usize = 2; + /// A [`ResolutionReport`] the user has confirmed. /// /// **This type is how DATA-FLOW §9.1 stops being a rule and starts being a fact.** There is no @@ -1145,6 +1154,7 @@ mod tests { forced: Vec::new(), yanked: Vec::new(), raw: String::new(), + declared_python: Default::default(), }; assert!(!report.is_clean()); } @@ -1164,6 +1174,7 @@ mod tests { forced: Vec::new(), yanked: Vec::new(), raw: String::new(), + declared_python: Default::default(), }; assert!(report.is_clean()); let pinned = report.pinned_set(); diff --git a/crates/pipdock-core/src/plan/preview.rs b/crates/pipdock-core/src/plan/preview.rs index c88400e..00cb9c5 100644 --- a/crates/pipdock-core/src/plan/preview.rs +++ b/crates/pipdock-core/src/plan/preview.rs @@ -308,6 +308,7 @@ mod tests { forced: Vec::new(), yanked: Vec::new(), raw: String::new(), + declared_python: Default::default(), } } diff --git a/crates/pipdock-core/src/release.rs b/crates/pipdock-core/src/release.rs new file mode 100644 index 0000000..2557c76 --- /dev/null +++ b/crates/pipdock-core/src/release.rs @@ -0,0 +1,235 @@ +//! What a release requires of the interpreter — the half of PipDock's own `Requires-Python` +//! enforcement an engine's plan cannot always supply (hard invariant 10). +//! +//! The engines disagree, which is why PipDock checks at all: pip filters candidates by +//! `Requires-Python` and refuses an incompatible exact pin with the same "No matching +//! distribution" it gives a typo; uv honours the lower bound and **ignores an upper bound**, so it +//! plans `scipy==1.7.3` (`<3.10`) on Python 3.14 and the install fails later, building from +//! source. pip's report carries each planned release's `requires_python`; uv's plan is text and +//! carries nothing, so for uv — and for an exact pin under either engine, checked before the +//! engine runs — the answer comes from PyPI's per-release JSON. +//! +//! **Unknown never blocks.** A release PyPI does not know (a private index), a network that is +//! down, a specifier that will not parse: each is "do not block", because refusing an installable +//! package on PipDock's uncertainty is the worse failure, and the engine still judges. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; + +use crate::model::{PkgName, Version}; +use crate::store::Store; + +/// How many lookups may be in flight at once. A plan for "update everything" can name fifty +/// releases; PyPI is a CDN, but fifty simultaneous connections from one desktop app is rude. +pub const LOOKUP_CONCURRENCY: usize = 8; + +/// Per-request watchdog. A lookup that takes longer is treated as unknown, which never blocks. +pub const LOOKUP_TIMEOUT: Duration = Duration::from_secs(15); + +/// Where `Requires-Python` of a release comes from. +#[async_trait] +pub trait ReleaseMeta: Send + Sync + std::fmt::Debug { + /// `Requires-Python` of `name==version`; `None` when it declares none **or nobody knows** — + /// both mean "do not block". + async fn requires_python(&self, name: &PkgName, version: &Version) -> Option; +} + +/// PyPI's per-release JSON, remembered in the store's `release_meta` table **forever**: an +/// uploaded release's metadata never changes, so a row is never refreshed. Only an answer is +/// remembered; a failure is asked again next time. +#[derive(Debug, Clone)] +pub struct PyPiReleases { + app_data: PathBuf, +} + +impl PyPiReleases { + /// Lookups cached under `app_data`'s store. + #[must_use] + pub fn new(app_data: impl Into) -> Self { + Self { + app_data: app_data.into(), + } + } + + /// `Some(answer)` when this release has been asked about before. + /// + /// The store is opened for this one read and dropped: `Store` is not `Sync`, and holding it + /// across the network call would make every caller's future not `Send`. + fn cached(&self, name: &PkgName, version: &Version) -> Option> { + let store = Store::open(&self.app_data).ok()?; + store + .conn() + .query_row( + "SELECT requires_python FROM release_meta WHERE normalized = ?1 AND version = ?2", + rusqlite::params![name.as_str(), version.0], + |r| r.get::<_, Option>(0), + ) + .ok() + } + + fn remember(&self, name: &PkgName, version: &Version, requires_python: Option<&str>) { + // A cache that cannot be written only costs the next run a request. + if let Ok(store) = Store::open(&self.app_data) { + let _ = store.conn().execute( + "INSERT INTO release_meta (normalized, version, requires_python) + VALUES (?1, ?2, ?3) + ON CONFLICT (normalized, version) DO UPDATE SET + requires_python = excluded.requires_python", + rusqlite::params![name.as_str(), version.0, requires_python], + ); + } + } +} + +#[async_trait] +impl ReleaseMeta for PyPiReleases { + async fn requires_python(&self, name: &PkgName, version: &Version) -> Option { + if let Some(answer) = self.cached(name, version) { + return answer; + } + let answer = fetch(name, version).await?; + self.remember(name, version, answer.as_deref()); + answer + } +} + +/// `GET https://pypi.org/pypi/{name}/{version}/json` → `info.requires_python`. +/// +/// `Some(None)` is a real answer — the release declares no constraint. `None` is "could not +/// find out": not on PyPI, unreachable, slow, unreadable. +async fn fetch(name: &PkgName, version: &Version) -> Option> { + // The version goes into a URL path; only a PEP 440 version is allowed to. + if !crate::pep440::is_version(&version.0) { + return None; + } + let url = format!("https://pypi.org/pypi/{}/{}/json", name.as_str(), version.0); + let response = crate::index::shared_client() + .ok()? + .get(&url) + .timeout(LOOKUP_TIMEOUT) + .send() + .await + .ok()?; + if !response.status().is_success() { + return None; + } + let doc: serde_json::Value = response.json().await.ok()?; + Some( + doc.get("info") + .and_then(|i| i.get("requires_python")) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_owned), + ) +} + +/// Look up many releases at once, at most [`LOOKUP_CONCURRENCY`] in flight. Releases with no +/// known constraint are absent from the answer. +pub async fn requires_python_of( + meta: &Arc, + releases: Vec<(PkgName, Version)>, +) -> BTreeMap<(PkgName, String), String> { + let gate = Arc::new(tokio::sync::Semaphore::new(LOOKUP_CONCURRENCY)); + let mut set = tokio::task::JoinSet::new(); + for (name, version) in releases { + let meta = Arc::clone(meta); + let gate = Arc::clone(&gate); + set.spawn(async move { + let _permit = gate.acquire_owned().await.ok(); + let answer = meta.requires_python(&name, &version).await; + (name, version, answer) + }); + } + let mut out = BTreeMap::new(); + while let Some(joined) = set.join_next().await { + if let Ok((name, version, Some(requires))) = joined { + out.insert((name, version.0), requires); + } + } + out +} + +/// `Some` when `requires_python` excludes the interpreter at `python` (e.g. `"3.14.7"`). +/// +/// An unparseable interpreter version or specifier is not a mismatch — see the module note. +#[must_use] +pub fn mismatch( + pkg: &PkgName, + version: &Version, + requires_python: Option<&str>, + python: &str, +) -> Option { + let interpreter = crate::compat::PyVersion::parse(python).ok()?; + match crate::compat::check(requires_python, &interpreter) { + crate::compat::Compatibility::RequiresPython { required, .. } => { + Some(crate::plan::PythonMismatch { + pkg: pkg.clone(), + version: version.clone(), + requires_python: required, + python: python.to_owned(), + }) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pkg(n: &str) -> PkgName { + PkgName::parse(n).unwrap() + } + + #[test] + fn a_capped_release_on_a_newer_python_is_a_mismatch() { + let m = mismatch( + &pkg("scipy"), + &Version("1.7.3".into()), + Some(">=3.7,<3.10"), + "3.14.7", + ) + .expect("excluded"); + assert_eq!(m.requires_python, ">=3.7,<3.10"); + assert_eq!(m.python, "3.14.7"); + assert!( + mismatch( + &pkg("scipy"), + &Version("1.16.0".into()), + Some(">=3.11"), + "3.14.7" + ) + .is_none() + ); + } + + #[test] + fn what_cannot_be_read_never_blocks() { + let v = Version("1.0".into()); + assert!(mismatch(&pkg("a"), &v, None, "3.14.7").is_none()); + assert!(mismatch(&pkg("a"), &v, Some(""), "3.14.7").is_none()); + assert!(mismatch(&pkg("a"), &v, Some("not a specifier"), "3.14.7").is_none()); + assert!(mismatch(&pkg("a"), &v, Some("<3.10"), "not a version").is_none()); + } + + #[test] + fn an_answer_is_remembered_and_a_failure_is_not() { + let dir = std::env::temp_dir().join(format!("pd-release-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let meta = PyPiReleases::new(&dir); + let (name, v) = (pkg("scipy"), Version("1.7.3".into())); + assert_eq!(meta.cached(&name, &v), None, "never asked"); + meta.remember(&name, &v, Some(">=3.7,<3.10")); + assert_eq!(meta.cached(&name, &v), Some(Some(">=3.7,<3.10".to_owned()))); + // "Declares nothing" is an answer too, and is not asked again. + let free = Version("1.0".into()); + meta.remember(&name, &free, None); + assert_eq!(meta.cached(&name, &free), Some(None)); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/pipdock-core/src/store/mod.rs b/crates/pipdock-core/src/store/mod.rs index 26eaf39..c383ca1 100644 --- a/crates/pipdock-core/src/store/mod.rs +++ b/crates/pipdock-core/src/store/mod.rs @@ -5,9 +5,10 @@ //! lifetime — deleting the app data folder is documented as a complete reset (SECURITY §8), and //! that promise is easier to keep with one file than four. //! -//! Migrations are forward-only and idempotent, applied on every open. There is no downgrade path: -//! an older PipDock opening a newer database is a situation the updater does not create, and -//! pretending to support it would mean writing migrations nobody can test. +//! Migrations are forward-only and idempotent, and run only when the database's `user_version` is +//! older than this build's. There is no downgrade path, but an older PipDock opening a newer +//! database must not *lower* the version either — it used to set it unconditionally, so the next +//! newer build re-ran every migration. use std::path::{Path, PathBuf}; @@ -19,7 +20,7 @@ use crate::errors::{Code, PdError, Result}; pub const DB_FILE: &str = "index.db"; /// Schema version this build expects. Bump when adding a migration. -pub const SCHEMA_VERSION: i64 = 3; +pub const SCHEMA_VERSION: i64 = 4; /// An open store. #[derive(Debug)] @@ -117,10 +118,23 @@ impl Store { PdError::new(Code::IntUnexpected, format!("store migration failed: {e}")) }; + // Per connection, so every open; everything below is per database, and the database + // remembers it. Every open used to run the whole batch — a `journal_mode` write and eight + // `CREATE TABLE`s — on a store some paths open twice per metadata lookup (M38). + self.conn + .execute_batch("PRAGMA foreign_keys = ON;") + .map_err(sql)?; + let current: i64 = self + .conn + .pragma_query_value(None, "user_version", |r| r.get(0)) + .map_err(sql)?; + if current >= SCHEMA_VERSION { + return Ok(()); + } + self.conn .execute_batch( "PRAGMA journal_mode = WAL; - PRAGMA foreign_keys = ON; CREATE TABLE IF NOT EXISTS names ( name TEXT NOT NULL, @@ -168,6 +182,16 @@ impl Store { CREATE TABLE IF NOT EXISTS kv ( key TEXT PRIMARY KEY, value TEXT NOT NULL + ); + + -- v4. `Requires-Python` of one release, as PyPI publishes it (crate::release). + -- An uploaded release's metadata never changes, so a row is never refreshed; + -- NULL is the answer \"declares none\", not \"unknown\", which is never stored. + CREATE TABLE IF NOT EXISTS release_meta ( + normalized TEXT NOT NULL, + version TEXT NOT NULL, + requires_python TEXT, + PRIMARY KEY (normalized, version) );", ) .map_err(sql)?; @@ -397,4 +421,41 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + + #[test] + fn an_older_build_never_lowers_the_version() { + let dir = std::env::temp_dir().join(format!("pd-store-newer-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let store = Store::open(&dir).expect("open"); + store + .conn() + .pragma_update(None, "user_version", SCHEMA_VERSION + 1) + .expect("a newer build was here"); + drop(store); + + let reopened = Store::open(&dir).expect("this build opens it"); + let version: i64 = reopened + .conn() + .pragma_query_value(None, "user_version", |r| r.get(0)) + .expect("version"); + assert_eq!(version, SCHEMA_VERSION + 1); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_v3_store_gains_the_release_cache() { + let store = Store::in_memory().expect("open"); + store + .conn() + .execute_batch("DROP TABLE release_meta; PRAGMA user_version = 3;") + .expect("rewind to v3"); + store.migrate().expect("migrate"); + store + .conn() + .execute( + "INSERT INTO release_meta (normalized, version, requires_python) VALUES ('a', '1', NULL)", + [], + ) + .expect("the v4 table exists"); + } } diff --git a/crates/pipdock-core/tests/execution.rs b/crates/pipdock-core/tests/execution.rs index 1f4c6a6..e0db859 100644 --- a/crates/pipdock-core/tests/execution.rs +++ b/crates/pipdock-core/tests/execution.rs @@ -125,6 +125,7 @@ impl Engine for FakeEngine { forced: Vec::new(), yanked: Vec::new(), raw: String::new(), + declared_python: Default::default(), }) } async fn install( @@ -291,6 +292,7 @@ fn report(packages: &[(&str, &str)]) -> ResolutionReport { forced: Vec::new(), yanked: Vec::new(), raw: String::new(), + declared_python: Default::default(), } } diff --git a/crates/pipdock-core/tests/flows.rs b/crates/pipdock-core/tests/flows.rs index a6f3980..5005eb9 100644 --- a/crates/pipdock-core/tests/flows.rs +++ b/crates/pipdock-core/tests/flows.rs @@ -47,6 +47,11 @@ struct World { installed: BTreeMap, /// The newest release the index offers, per package. latest: BTreeMap, + /// Older releases the index also offers, newest first — what a resolver falls back to when + /// the latest is ruled out. + older: BTreeMap>, + /// Which engine this pretends to be; the flow asks PyPI about uv's plans, not pip's. + engine: Option, /// What a package's newest release needs at least, per dependency: `httpcore` 1.0.9 needs /// `h11>=0.16`. needs: BTreeMap>, @@ -63,10 +68,6 @@ struct World { /// What one resolve was asked: the request, what the flow changed about the guard set, and /// whether the PEP 668 override applied. -#[expect( - dead_code, - reason = "recorded whole, for whichever part a test asserts on" -)] struct ResolveCall { req: PlanRequest, adjust: GuardAdjust, @@ -101,6 +102,19 @@ impl Scripted { self } + fn also_offering(self, name: &str, older: &[&str]) -> Self { + self.world().older.insert( + name.to_owned(), + older.iter().map(|v| (*v).to_owned()).collect(), + ); + self + } + + fn pretending_to_be(self, id: EngineId) -> Self { + self.world().engine = Some(id); + self + } + fn needing(self, name: &str, dep: &str, at_least: &str) -> Self { self.world() .needs @@ -152,7 +166,7 @@ fn env() -> PyEnv { #[async_trait] impl Engine for Scripted { fn id(&self) -> EngineId { - EngineId::Pip + self.world().engine.unwrap_or(EngineId::Pip) } async fn info(&self, _env: &PyEnv) -> EngineInfo { @@ -238,15 +252,41 @@ impl Engine for Scripted { } } + let excluded = |name: &str, version: &str| { + ctx.adjust + .exclude + .iter() + .any(|e| e.name.as_str() == name && e.version.0 == version) + }; let mut changes = Vec::new(); + for spec in &req.installs { + let Some(version) = spec + .version_req + .as_deref() + .and_then(|v| v.strip_prefix("==")) + else { + continue; + }; + changes.push(Change { + name: spec.name.clone(), + from: None, + to: Version(version.to_owned()), + kind: ChangeKind::NewInstall, + }); + } for name in &req.upgrades { if pinned.contains(&name.as_str()) { continue; } - let (Some(from), Some(to)) = ( - world.installed.get(name.as_str()), - world.latest.get(name.as_str()), - ) else { + let Some(from) = world.installed.get(name.as_str()) else { + continue; + }; + let candidates = world + .latest + .get(name.as_str()) + .into_iter() + .chain(world.older.get(name.as_str()).into_iter().flatten()); + let Some(to) = candidates.into_iter().find(|v| !excluded(name.as_str(), v)) else { continue; }; let capped = guard @@ -861,3 +901,187 @@ async fn force_latest_forces_and_names_what_it_breaks() { assert!(last.release.contains(&pkg("httpx"))); assert!(last.release.contains(&pkg("h11"))); } + +/// `Requires-Python` per release, as PyPI would answer; anything absent is unknown. +#[derive(Debug, Default)] +struct Releases { + known: BTreeMap<(String, String), String>, + asked: std::sync::atomic::AtomicUsize, +} + +impl Releases { + fn with(mut self, name: &str, version: &str, requires: &str) -> Self { + self.known + .insert((name.to_owned(), version.to_owned()), requires.to_owned()); + self + } +} + +#[async_trait] +impl pipdock_core::release::ReleaseMeta for Releases { + async fn requires_python(&self, name: &PkgName, version: &Version) -> Option { + self.asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.known + .get(&(name.to_string(), version.0.clone())) + .cloned() + } +} + +fn checking(releases: &Arc) -> FlowOptions { + FlowOptions { + releases: Some(Arc::clone(releases) as Arc), + ..FlowOptions::default() + } +} + +fn install(specs: &[&str]) -> Intent { + Intent::Install { + specs: specs.iter().map(|s| (*s).to_owned()).collect(), + } +} + +#[tokio::test] +async fn an_exact_pin_this_python_cannot_install_is_pd_pkg_001_before_any_engine_runs() { + // C2. pip answered this with the "No matching distribution" it gives a typo, and uv planned + // it and failed at install time, building scipy from source. PD-PKG-001 existed in the + // catalog and nothing ever raised it. + let releases = Arc::new(Releases::default().with("scipy", "1.7.3", ">=3.7,<3.10")); + for engine_id in [EngineId::Pip, EngineId::Uv] { + let engine = Scripted::default().pretending_to_be(engine_id); + let Err(err) = UpdateFlow::start( + env(), + Box::new(engine.clone()), + &install(&["scipy==1.7.3"]), + &[], + checking(&releases), + ) + .await + else { + panic!("{engine_id:?}: an install this Python cannot run must be refused"); + }; + assert_eq!(err.code, Code::PkgRequiresPython, "{engine_id:?}"); + assert!(err.message.contains(">=3.7,<3.10") && err.message.contains("3.12.4")); + assert!( + engine.world().resolves.is_empty(), + "refused before the engine ran" + ); + } +} + +#[tokio::test] +async fn one_pin_this_python_cannot_install_does_not_cost_the_others() { + let releases = Arc::new(Releases::default().with("scipy", "1.7.3", ">=3.7,<3.10")); + let engine = Scripted::default(); + let (_, step) = UpdateFlow::start( + env(), + Box::new(engine.clone()), + &install(&["scipy==1.7.3", "idna==3.10"]), + &[], + checking(&releases), + ) + .await + .expect("start"); + let FlowStep::NeedsConfirm { report } = step else { + panic!("the rest of the request is confirmable: {step:?}"); + }; + let planned: Vec = report.changes.iter().map(|c| c.name.to_string()).collect(); + assert_eq!(planned, ["idna"]); + assert_eq!(report.incompatible.len(), 1); + assert_eq!(report.incompatible[0].pkg.as_str(), "scipy"); + assert_eq!(report.incompatible[0].requires_python, ">=3.7,<3.10"); + let world = engine.world(); + let asked: Vec = world.resolves[0] + .req + .installs + .iter() + .map(|s| s.name.to_string()) + .collect(); + assert_eq!(asked, ["idna"], "scipy never reached the engine"); +} + +#[tokio::test] +async fn a_release_uv_planned_past_its_python_cap_is_ruled_out_and_the_engine_asked_again() { + // uv honours a lower Requires-Python bound and ignores an upper one, so it plans numba + // 0.62.0 (<3.14) on 3.14. PipDock rules that release out by name and uv picks again. + let releases = Arc::new( + Releases::default() + .with("numba", "0.62.0", ">=3.10,<3.14") + .with("numba", "0.61.2", ">=3.10,<3.15"), + ); + let engine = Scripted::default() + .pretending_to_be(EngineId::Uv) + .with("numba", "0.60.0", &[]) + .offering("numba", "0.62.0") + .also_offering("numba", &["0.61.2"]); + let mut py314 = env(); + py314.python_version = "3.14.7".into(); + let (_, step) = UpdateFlow::start( + py314, + Box::new(engine.clone()), + &update_all(), + &[], + checking(&releases), + ) + .await + .expect("start"); + let FlowStep::NeedsConfirm { report } = step else { + panic!("an older compatible release is a clean plan, not a held-back row: {step:?}"); + }; + assert_eq!(report.changes.len(), 1); + assert_eq!(report.changes[0].to.0, "0.61.2"); + assert_eq!( + report.incompatible.len(), + 1, + "and the preview says what was ruled out" + ); + assert_eq!(report.incompatible[0].version.0, "0.62.0"); + let world = engine.world(); + assert_eq!(world.resolves.len(), 2); + assert_eq!(world.resolves[1].adjust.exclude[0].version.0, "0.62.0"); +} + +#[tokio::test] +async fn pip_s_own_answer_is_used_and_pypi_is_not_asked_about_its_plan() { + let releases = Arc::new(Releases::default()); + let engine = Scripted::default() + .with("idna", "3.4", &[]) + .offering("idna", "3.10"); + let (_, step) = UpdateFlow::start( + env(), + Box::new(engine.clone()), + &update_all(), + &[], + checking(&releases), + ) + .await + .expect("start"); + assert!(matches!(step, FlowStep::NeedsConfirm { .. }), "{step:?}"); + assert_eq!( + releases.asked.load(std::sync::atomic::Ordering::SeqCst), + 0, + "nothing exact was pinned and pip's plan says what it requires" + ); +} + +#[tokio::test] +async fn a_requires_python_nobody_knows_never_blocks() { + // A private index, a network that is down: refusing an installable package on PipDock's + // uncertainty is the worse failure, and the engine still judges. + let releases = Arc::new(Releases::default()); + let engine = Scripted::default().pretending_to_be(EngineId::Uv); + let (_, step) = UpdateFlow::start( + env(), + Box::new(engine), + &install(&["internal-tool==2.0"]), + &[], + checking(&releases), + ) + .await + .expect("start"); + let FlowStep::NeedsConfirm { report } = step else { + panic!("{step:?}"); + }; + assert_eq!(report.changes.len(), 1); + assert!(report.incompatible.is_empty()); + assert!(releases.asked.load(std::sync::atomic::Ordering::SeqCst) >= 1); +} diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 369e564..9806c2d 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -665,6 +665,9 @@ pub async fn plan_resolve( let opts = flow::FlowOptions { cancel, allow_externally_managed, + releases: Some(std::sync::Arc::new( + pipdock_core::release::PyPiReleases::new(state.app_data.clone()), + )), ..flow::FlowOptions::default() }; match flow::UpdateFlow::start(env, engine, &intent, &env_pins, opts).await {