From 30950833c943c6b6b48d204ab0027f38af356f5c Mon Sep 17 00:00:00 2001
From: Florian Klink <flokli@flokli.de>
Date: Mon, 22 Apr 2024 15:18:14 +0300
Subject: feat(tvix/glue/store_io): have KnownPaths track fetches too

Have fetcher builtins call queue_fetch() whenever they don't need to
fetch something immediately, and teach TvixStoreIO::store_path_to_node
on how to look up (and call ingest_and persist on our Fetcher).

Change-Id: Id4bd9d639fac9e4bee20c0b1c584148740b15c2f
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11501
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
---
 tvix/glue/src/tvix_store_io.rs | 32 +++++++++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)

(limited to 'tvix/glue/src/tvix_store_io.rs')

diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs
index 0994c44dfa..c7133e1d10 100644
--- a/tvix/glue/src/tvix_store_io.rs
+++ b/tvix/glue/src/tvix_store_io.rs
@@ -15,7 +15,7 @@ use std::{
     sync::Arc,
 };
 use tokio_util::io::SyncIoBridge;
-use tracing::{error, instrument, warn, Level};
+use tracing::{error, info, instrument, warn, Level};
 use tvix_build::buildservice::BuildService;
 use tvix_castore::proto::node::Node;
 use tvix_eval::{EvalIO, FileType, StdIO};
@@ -61,6 +61,7 @@ pub struct TvixStoreIO {
     pub(crate) fetcher:
         Fetcher<Arc<dyn BlobService>, Arc<dyn DirectoryService>, Arc<dyn PathInfoService>>,
 
+    // Paths known how to produce, by building or fetching.
     pub(crate) known_paths: RefCell<KnownPaths>,
 }
 
@@ -121,8 +122,31 @@ impl TvixStoreIO {
             // it for things like <nixpkgs> pointing to a store path.
             // In the future, these things will (need to) have PathInfo.
             None => {
-                // The store path doesn't exist yet, so we need to build it.
-                warn!("triggering build");
+                // The store path doesn't exist yet, so we need to fetch or build it.
+                // We check for fetches first, as we might have both native
+                // fetchers and FODs in KnownPaths, and prefer the former.
+
+                let maybe_fetch = self
+                    .known_paths
+                    .borrow()
+                    .get_fetch_for_output_path(store_path);
+
+                if let Some((name, fetch)) = maybe_fetch {
+                    info!(?fetch, "triggering lazy fetch");
+                    let (sp, root_node) = self
+                        .fetcher
+                        .ingest_and_persist(&name, fetch)
+                        .await
+                        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+
+                    debug_assert_eq!(
+                        sp.to_string(),
+                        store_path.to_string(),
+                        "store path returned from fetcher should match"
+                    );
+
+                    return Ok(Some(root_node));
+                }
 
                 // Look up the derivation for this output path.
                 let (drv_path, drv) = {
@@ -140,6 +164,8 @@ impl TvixStoreIO {
                     }
                 };
 
+                warn!("triggering build");
+
                 // derivation_to_build_request needs castore nodes for all inputs.
                 // Provide them, which means, here is where we recursively build
                 // all dependencies.
-- 
cgit 1.4.1