refactor: move PyPI resolution and installation into the command dispatcher

Cargo.toml

@@ -105,6 +105,7 @@ pixi_compute_cache_dirs = { path = "crates/pixi_compute_cache_dirs" }
 pixi_compute_engine = { path = "crates/pixi_compute_engine" }
 pixi_compute_env_vars = { path = "crates/pixi_compute_env_vars" }
 pixi_compute_network = { path = "crates/pixi_compute_network" }
+pixi_compute_pypi = { path = "crates/pixi_compute_pypi" }
 pixi_compute_reporters = { path = "crates/pixi_compute_reporters" }
 pixi_compute_sources = { path = "crates/pixi_compute_sources" }
 pixi_config = { path = "crates/pixi_config" }
@@ -133,6 +134,7 @@ pixi_url = { path = "crates/pixi_url" }
 pixi_utils = { path = "crates/pixi_utils", default-features = false }
 pixi_uv_context = { path = "crates/pixi_uv_context" }
 pixi_uv_conversions = { path = "crates/pixi_uv_conversions" }
+pixi_uv_reporter = { path = "crates/pixi_uv_reporter" }
 pixi_variant = { path = "crates/pixi_variant" }
 pypi_mapping = { path = "crates/pypi_mapping" }
 pypi_modifiers = { path = "crates/pypi_modifiers" }

crates/pixi_command_dispatcher/Cargo.toml

@@ -80,7 +80,6 @@ tempfile = { workspace = true }
 text_trees = { workspace = true }
 tokio = { workspace = true, features = ["macros"] }
 
-
 [features]
 slow_integration_tests = []
 

crates/pixi_command_dispatcher/src/command_dispatcher/builder.rs

@@ -44,6 +44,18 @@ use tokio::sync::Semaphore;
 
 #[derive(Default)]
 pub struct CommandDispatcherBuilder {
+    /// Opaque values registered into the engine's [`DataStore`]
+    /// (`pixi_compute_engine::DataStore`) at `finish()`. This lets crates
+    /// that build on the compute engine (e.g. the PyPI pipeline) receive
+    /// their shared resources and reporters without this crate knowing
+    /// about them.
+    engine_data: Vec<
+        Box<
+            dyn FnOnce(
+                pixi_compute_engine::ComputeEngineBuilder,
+            ) -> pixi_compute_engine::ComputeEngineBuilder,
+        >,
+    >,
     gateway: Option<Gateway>,
     root_dir: Option<AbsPresumedDirPathBuf>,
     git_resolver: Option<GitResolver>,
@@ -83,6 +95,17 @@ pub struct CommandDispatcherBuilder {
 }
 
 impl CommandDispatcherBuilder {
+    /// Register an arbitrary value into the compute engine's data store.
+    ///
+    /// Crates that build on the compute engine (rather than on this crate)
+    /// use this to make their shared resources and reporters available to
+    /// their compute bodies and extension traits.
+    pub fn with_engine_data<T: Send + Sync + 'static>(mut self, value: T) -> Self {
+        self.engine_data
+            .push(Box::new(move |builder| builder.with_data(value)));
+        self
+    }
+
     /// Sets the cache directories to use.
     pub fn with_cache_dirs(self, cache_dirs: CacheDirs) -> Self {
         Self {
@@ -518,6 +541,10 @@ impl CommandDispatcherBuilder {
         if let Some(sem) = data.io_concurrency_semaphore.clone() {
             engine_builder = engine_builder.with_data(IoConcurrencySemaphore(sem));
         }
+        // Register the opaque values supplied through `with_engine_data`.
+        for register in self.engine_data {
+            engine_builder = register(engine_builder);
+        }
         let engine = engine_builder.build();
 
         // Inject engine-wide configuration values that Keys read through

crates/pixi_command_dispatcher/src/compute_data.rs

@@ -174,25 +174,10 @@ impl HasAllowExecuteLinkScripts for DataStore {
 }
 
 /// Configured allow/disallow preferences for installation link methods,
-/// stored in the [`DataStore`] keyed by `TypeId`. Mirrors the fields on
-/// [`rattler::install::LinkOptions`] but is `Copy`/`Clone`/`Debug` so it
-/// can live in the data store.
-#[derive(Copy, Clone, Debug, Default)]
-pub struct AllowLinkOptions {
-    pub allow_symbolic_links: Option<bool>,
-    pub allow_hard_links: Option<bool>,
-    pub allow_ref_links: Option<bool>,
-}
-
-impl From<AllowLinkOptions> for rattler::install::LinkOptions {
-    fn from(opts: AllowLinkOptions) -> Self {
-        rattler::install::LinkOptions {
-            allow_symbolic_links: opts.allow_symbolic_links,
-            allow_hard_links: opts.allow_hard_links,
-            allow_ref_links: opts.allow_ref_links,
-        }
-    }
-}
+/// stored in the [`DataStore`] keyed by `TypeId`. The type lives in
+/// `pixi_utils` so the PyPI installation pipeline can read the same value
+/// without depending on this crate.
+pub use pixi_utils::link_options::AllowLinkOptions;
 
 /// Access the configured link-method preferences.
 pub trait HasAllowLinkOptions {
@@ -201,10 +186,15 @@ pub trait HasAllowLinkOptions {
 
 impl HasAllowLinkOptions for DataStore {
     fn allow_link_options(&self) -> rattler::install::LinkOptions {
-        self.try_get::<AllowLinkOptions>()
+        let opts = self
+            .try_get::<AllowLinkOptions>()
             .copied()
-            .unwrap_or_default()
-            .into()
+            .unwrap_or_default();
+        rattler::install::LinkOptions {
+            allow_symbolic_links: opts.allow_symbolic_links,
+            allow_hard_links: opts.allow_hard_links,
+            allow_ref_links: opts.allow_ref_links,
+        }
     }
 }
 

crates/pixi_compute_engine/src/engine.rs

@@ -321,6 +321,17 @@ impl ComputeEngine {
     /// the injected value changes. Use
     /// [`ComputeCtx::compute`](crate::ComputeCtx::compute) there
     /// instead.
+    /// Returns the engine-wide shared data store.
+    ///
+    /// The store is populated at engine construction and immutable for the
+    /// engine's lifetime; this accessor lets code outside a Key's compute
+    /// body (e.g. extension-trait wrappers) read the same shared resources
+    /// that compute bodies reach through
+    /// [`ComputeCtx::global_data`](crate::ComputeCtx::global_data).
+    pub fn global_data(&self) -> &DataStore {
+        &self.inner.global_data
+    }
+
     pub fn read<K: InjectedKey>(&self, key: &K) -> Option<K::Value> {
         match self.inner.graph.lookup::<K>(key) {
             Some(Lookup::Completed(value)) => Some(value),

crates/pixi_compute_pypi/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+authors.workspace = true
+description = "Compute-engine integration for pixi's PyPI resolve and install pipeline"
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+name = "pixi_compute_pypi"
+readme.workspace = true
+repository.workspace = true
+version = "0.1.0"
+
+[dependencies]
+indexmap = { workspace = true }
+miette = { workspace = true }
+once_cell = { workspace = true }
+ordermap = { workspace = true }
+
+pixi_compute_engine = { workspace = true }
+pixi_compute_reporters = { workspace = true }
+pixi_compute_sources = { workspace = true }
+pixi_install_pypi = { workspace = true }
+pixi_manifest = { workspace = true, features = ["rattler_lock"] }
+pixi_pypi_spec = { workspace = true }
+pixi_python_status = { workspace = true }
+pixi_record = { workspace = true }
+pixi_utils = { workspace = true, default-features = false }
+pixi_uv_context = { workspace = true }
+pixi_uv_conversions = { workspace = true }
+pixi_uv_reporter = { workspace = true }
+
+rattler_lock = { workspace = true }
+
+uv-normalize = { workspace = true }

crates/pixi_compute_pypi/src/data.rs

@@ -0,0 +1,55 @@
+//! Engine-side accessors for the shared resources of the PyPI pipeline.
+
+use miette::miette;
+use once_cell::sync::OnceCell;
+use pixi_compute_engine::DataStore;
+use pixi_uv_context::UvResolutionContext;
+
+/// Lazily initialized [`UvResolutionContext`] registered in the engine's
+/// data store.
+///
+/// Constructing the context forces the (otherwise lazy) reqwest client,
+/// which loads the system certificate store — too expensive to do eagerly
+/// on every engine construction when most operations never touch PyPI. The
+/// initializer runs on first use and the result is memoized for the
+/// engine's lifetime.
+pub struct UvResolutionContextSource {
+    cell: OnceCell<UvResolutionContext>,
+    init: Box<dyn Fn() -> miette::Result<UvResolutionContext> + Send + Sync>,
+}
+
+impl UvResolutionContextSource {
+    pub fn new(
+        init: impl Fn() -> miette::Result<UvResolutionContext> + Send + Sync + 'static,
+    ) -> Self {
+        Self {
+            cell: OnceCell::new(),
+            init: Box::new(init),
+        }
+    }
+
+    /// Returns the context, initializing it on first use.
+    pub fn get(&self) -> miette::Result<&UvResolutionContext> {
+        self.cell.get_or_try_init(|| (self.init)())
+    }
+}
+
+/// Access the shared uv context (cache, concurrency, http settings) from
+/// global data.
+pub trait HasUvResolutionContext {
+    fn uv_resolution_context(&self) -> miette::Result<&UvResolutionContext>;
+}
+
+impl HasUvResolutionContext for DataStore {
+    fn uv_resolution_context(&self) -> miette::Result<&UvResolutionContext> {
+        self.try_get::<UvResolutionContextSource>()
+            .ok_or_else(|| {
+                miette!(
+                    "no `UvResolutionContextSource` was registered on the compute engine; \
+                     register one through `CommandDispatcherBuilder::with_engine_data` before \
+                     performing PyPI operations"
+                )
+            })?
+            .get()
+    }
+}