refactor(tvix/castore/import): make module, split off fs and error r/7981

Move error types and filesystem-specific functions to a separate file,
and keep the fs:: namespace in public exports.

Change-Id: I5e9e83ad78d9aea38553fafc293d3e4f8c31a8c1
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11486
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Autosubmit: flokli <flokli@flokli.de>

7 files changed, 225 insertions, 197 deletions

diff --git a/tvix/castore/src/import/error.rs b/tvix/castore/src/import/error.rs
new file mode 100644
index 0000000000..15dd0664de
--- /dev/null
+++ b/tvix/castore/src/import/error.rs
@@ -0,0 +1,39 @@
+use std::{fs::FileType, path::PathBuf};
+
+use crate::Error as CastoreError;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("failed to upload directory at {0}: {1}")]
+    UploadDirectoryError(PathBuf, CastoreError),
+
+    #[error("invalid encoding encountered for entry {0:?}")]
+    InvalidEncoding(PathBuf),
+
+    #[error("unable to stat {0}: {1}")]
+    UnableToStat(PathBuf, std::io::Error),
+
+    #[error("unable to open {0}: {1}")]
+    UnableToOpen(PathBuf, std::io::Error),
+
+    #[error("unable to read {0}: {1}")]
+    UnableToRead(PathBuf, std::io::Error),
+
+    #[error("unsupported file {0} type: {1:?}")]
+    UnsupportedFileType(PathBuf, FileType),
+}
+
+impl From<CastoreError> for Error {
+    fn from(value: CastoreError) -> Self {
+        match value {
+            CastoreError::InvalidRequest(_) => panic!("tvix bug"),
+            CastoreError::StorageError(_) => panic!("error"),
+        }
+    }
+}
+
+impl From<Error> for std::io::Error {
+    fn from(value: Error) -> Self {
+        std::io::Error::new(std::io::ErrorKind::Other, value)
+    }
+}
diff --git a/tvix/castore/src/import/fs.rs b/tvix/castore/src/import/fs.rs
new file mode 100644
index 0000000000..a87a0278da
--- /dev/null
+++ b/tvix/castore/src/import/fs.rs
@@ -0,0 +1,143 @@
+use std::os::unix::fs::MetadataExt;
+use std::os::unix::fs::PermissionsExt;
+use std::path::Path;
+
+use futures::stream::BoxStream;
+use tracing::instrument;
+use walkdir::DirEntry;
+use walkdir::WalkDir;
+
+use crate::blobservice::BlobService;
+use crate::directoryservice::DirectoryService;
+use crate::proto::node::Node;
+
+use super::ingest_entries;
+use super::upload_blob_at_path;
+use super::Error;
+use super::IngestionEntry;
+
+///! Imports that deal with a real filesystem.
+
+/// Ingests the contents at a given path into the tvix store, interacting with a [BlobService] and
+/// [DirectoryService]. It returns the root node or an error.
+///
+/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
+#[instrument(skip(blob_service, directory_service), fields(path), err)]
+pub async fn ingest_path<BS, DS, P>(
+    blob_service: BS,
+    directory_service: DS,
+    path: P,
+) -> Result<Node, Error>
+where
+    P: AsRef<Path> + std::fmt::Debug,
+    BS: BlobService + Clone,
+    DS: AsRef<dyn DirectoryService>,
+{
+    let entry_stream = walk_path_for_ingestion(blob_service, path.as_ref());
+    ingest_entries(directory_service, entry_stream).await
+}
+
+/// Walk the filesystem at a given path and returns a stream of ingestion entries.
+///
+/// This is how [`ingest_path`] assembles the set of entries to pass on [`ingest_entries`].
+/// This low-level function can be used if additional filtering or processing is required on the
+/// entries.
+///
+/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
+///
+/// This function will walk the filesystem using `walkdir` and will consume
+/// `O(#number of entries)` space.
+#[instrument(fields(path), skip(blob_service))]
+fn walk_path_for_ingestion<'a, BS>(
+    blob_service: BS,
+    path: &'a Path,
+) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
+where
+    BS: BlobService + Clone + 'a,
+{
+    let iter = WalkDir::new(path)
+        .follow_links(false)
+        .follow_root_links(false)
+        .contents_first(true)
+        .into_iter();
+
+    dir_entry_iter_to_ingestion_stream(blob_service, iter, path)
+}
+
+/// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries.
+/// This can then be fed into [ingest_entries] to ingest all the entries into the castore.
+///
+/// The root is the [Path] in the filesystem that is being ingested into the castore.
+pub fn dir_entry_iter_to_ingestion_stream<'a, BS, I>(
+    blob_service: BS,
+    iter: I,
+    root: &'a Path,
+) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
+where
+    BS: BlobService + Clone + 'a,
+    I: Iterator<Item = Result<DirEntry, walkdir::Error>> + Send + 'a,
+{
+    let prefix = root.parent().unwrap_or_else(|| Path::new(""));
+
+    let iter = iter.map(move |entry| match entry {
+        Ok(entry) => dir_entry_to_ingestion_entry(blob_service.clone(), &entry, prefix),
+        Err(error) => Err(Error::UnableToStat(
+            root.to_path_buf(),
+            error.into_io_error().expect("walkdir err must be some"),
+        )),
+    });
+
+    Box::pin(futures::stream::iter(iter))
+}
+
+/// Converts a [walkdir::DirEntry] into an [IngestionEntry], uploading blobs to the
+/// provided [BlobService].
+///
+/// The prefix path is stripped from the path of each entry. This is usually the parent path
+/// of the path being ingested so that the last element of the stream only has one component.
+fn dir_entry_to_ingestion_entry<'a, BS>(
+    blob_service: BS,
+    entry: &DirEntry,
+    prefix: &Path,
+) -> Result<IngestionEntry<'a>, Error>
+where
+    BS: BlobService + 'a,
+{
+    let file_type = entry.file_type();
+
+    let path = entry
+        .path()
+        .strip_prefix(prefix)
+        .expect("Tvix bug: failed to strip root path prefix")
+        .to_path_buf();
+
+    if file_type.is_dir() {
+        Ok(IngestionEntry::Dir { path })
+    } else if file_type.is_symlink() {
+        let target = std::fs::read_link(entry.path())
+            .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e))?;
+
+        Ok(IngestionEntry::Symlink { path, target })
+    } else if file_type.is_file() {
+        let metadata = entry
+            .metadata()
+            .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e.into()))?;
+
+        // TODO: In the future, for small files, hash right away and upload async.
+        let digest = Box::pin(upload_blob_at_path(
+            blob_service,
+            entry.path().to_path_buf(),
+        ));
+
+        Ok(IngestionEntry::Regular {
+            path,
+            size: metadata.size(),
+            // If it's executable by the user, it'll become executable.
+            // This matches nix's dump() function behaviour.
+            executable: metadata.permissions().mode() & 64 != 0,
+            digest,
+        })
+    } else {
+        Ok(IngestionEntry::Unknown { path, file_type })
+    }
+}
diff --git a/tvix/castore/src/import.rs b/tvix/castore/src/import/mod.rs
index a07cf71f6c..d2b1ee9ff7 100644
--- a/tvix/castore/src/import.rs
+++ b/tvix/castore/src/import/mod.rs
@@ -1,3 +1,10 @@
+//! Deals with ingesting contents into castore.
+//! The main library function here is [ingest_entries], receiving a stream of
+//! [IngestionEntry].
+//!
+//! Specific implementations, such as ingesting from the filesystem, live in
+//! child modules.
+
 use crate::blobservice::BlobService;
 use crate::directoryservice::DirectoryPutter;
 use crate::directoryservice::DirectoryService;
@@ -7,12 +14,9 @@ use crate::proto::DirectoryNode;
 use crate::proto::FileNode;
 use crate::proto::SymlinkNode;
 use crate::B3Digest;
-use crate::Error as CastoreError;
-use futures::stream::BoxStream;
 use futures::Future;
 use futures::{Stream, StreamExt};
 use std::fs::FileType;
-use std::os::unix::fs::MetadataExt;
 use std::pin::Pin;
 use tracing::Level;
 
@@ -21,200 +25,17 @@ use std::os::unix::ffi::OsStrExt;
 
 use std::{
     collections::HashMap,
-    fmt::Debug,
-    os::unix::prelude::PermissionsExt,
     path::{Path, PathBuf},
 };
 use tracing::instrument;
-use walkdir::DirEntry;
-use walkdir::WalkDir;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("failed to upload directory at {0}: {1}")]
-    UploadDirectoryError(PathBuf, CastoreError),
-
-    #[error("invalid encoding encountered for entry {0:?}")]
-    InvalidEncoding(PathBuf),
-
-    #[error("unable to stat {0}: {1}")]
-    UnableToStat(PathBuf, std::io::Error),
-
-    #[error("unable to open {0}: {1}")]
-    UnableToOpen(PathBuf, std::io::Error),
-
-    #[error("unable to read {0}: {1}")]
-    UnableToRead(PathBuf, std::io::Error),
-
-    #[error("unsupported file {0} type: {1:?}")]
-    UnsupportedFileType(PathBuf, FileType),
-}
-
-impl From<CastoreError> for Error {
-    fn from(value: CastoreError) -> Self {
-        match value {
-            CastoreError::InvalidRequest(_) => panic!("tvix bug"),
-            CastoreError::StorageError(_) => panic!("error"),
-        }
-    }
-}
-
-impl From<Error> for std::io::Error {
-    fn from(value: Error) -> Self {
-        std::io::Error::new(std::io::ErrorKind::Other, value)
-    }
-}
-
-/// Walk the filesystem at a given path and returns a stream of ingestion entries.
-///
-/// This is how [`ingest_path`] assembles the set of entries to pass on [`ingest_entries`].
-/// This low-level function can be used if additional filtering or processing is required on the
-/// entries.
-///
-/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
-///
-/// This function will walk the filesystem using `walkdir` and will consume
-/// `O(#number of entries)` space.
-#[instrument(fields(path), skip(blob_service))]
-fn walk_path_for_ingestion<'a, BS>(
-    blob_service: BS,
-    path: &'a Path,
-) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
-where
-    BS: BlobService + Clone + 'a,
-{
-    let iter = WalkDir::new(path)
-        .follow_links(false)
-        .follow_root_links(false)
-        .contents_first(true)
-        .into_iter();
-
-    dir_entry_iter_to_ingestion_stream(blob_service, iter, path)
-}
-
-/// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries.
-/// This can then be fed into [ingest_entries] to ingest all the entries into the castore.
-///
-/// The root is the [Path] in the filesystem that is being ingested into the castore.
-pub fn dir_entry_iter_to_ingestion_stream<'a, BS, I>(
-    blob_service: BS,
-    iter: I,
-    root: &'a Path,
-) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
-where
-    BS: BlobService + Clone + 'a,
-    I: Iterator<Item = Result<DirEntry, walkdir::Error>> + Send + 'a,
-{
-    let prefix = root.parent().unwrap_or_else(|| Path::new(""));
 
-    let iter = iter.map(move |entry| match entry {
-        Ok(entry) => dir_entry_to_ingestion_entry(blob_service.clone(), &entry, prefix),
-        Err(error) => Err(Error::UnableToStat(
-            root.to_path_buf(),
-            error.into_io_error().expect("walkdir err must be some"),
-        )),
-    });
+mod error;
+pub use error::Error;
 
-    Box::pin(futures::stream::iter(iter))
-}
+pub mod fs;
 
-/// Converts a [walkdir::DirEntry] into an [IngestionEntry], uploading blobs to the
-/// provided [BlobService].
-///
-/// The prefix path is stripped from the path of each entry. This is usually the parent path
-/// of the path being ingested so that the last element of the stream only has one component.
-fn dir_entry_to_ingestion_entry<'a, BS>(
-    blob_service: BS,
-    entry: &DirEntry,
-    prefix: &Path,
-) -> Result<IngestionEntry<'a>, Error>
-where
-    BS: BlobService + 'a,
-{
-    let file_type = entry.file_type();
-
-    let path = entry
-        .path()
-        .strip_prefix(prefix)
-        .expect("Tvix bug: failed to strip root path prefix")
-        .to_path_buf();
-
-    if file_type.is_dir() {
-        Ok(IngestionEntry::Dir { path })
-    } else if file_type.is_symlink() {
-        let target = std::fs::read_link(entry.path())
-            .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e))?;
-
-        Ok(IngestionEntry::Symlink { path, target })
-    } else if file_type.is_file() {
-        let metadata = entry
-            .metadata()
-            .map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e.into()))?;
-
-        // TODO: In the future, for small files, hash right away and upload async.
-        let digest = Box::pin(upload_blob_at_path(
-            blob_service,
-            entry.path().to_path_buf(),
-        ));
-
-        Ok(IngestionEntry::Regular {
-            path,
-            size: metadata.size(),
-            // If it's executable by the user, it'll become executable.
-            // This matches nix's dump() function behaviour.
-            executable: metadata.permissions().mode() & 64 != 0,
-            digest,
-        })
-    } else {
-        Ok(IngestionEntry::Unknown { path, file_type })
-    }
-}
-
-/// Uploads the file at the provided [Path] the the [BlobService].
-#[instrument(skip(blob_service), fields(path), err)]
-async fn upload_blob_at_path<BS>(blob_service: BS, path: PathBuf) -> Result<B3Digest, Error>
-where
-    BS: BlobService,
-{
-    let mut file = match tokio::fs::File::open(&path).await {
-        Ok(file) => file,
-        Err(e) => return Err(Error::UnableToRead(path, e)),
-    };
-
-    let mut writer = blob_service.open_write().await;
-
-    if let Err(e) = tokio::io::copy(&mut file, &mut writer).await {
-        return Err(Error::UnableToRead(path, e));
-    };
-
-    let digest = writer
-        .close()
-        .await
-        .map_err(|e| Error::UnableToRead(path, e))?;
-
-    Ok(digest)
-}
-
-/// Ingests the contents at a given path into the tvix store, interacting with a [BlobService] and
-/// [DirectoryService]. It returns the root node or an error.
-///
-/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
-#[instrument(skip(blob_service, directory_service), fields(path), err)]
-pub async fn ingest_path<BS, DS, P>(
-    blob_service: BS,
-    directory_service: DS,
-    path: P,
-) -> Result<Node, Error>
-where
-    P: AsRef<Path> + std::fmt::Debug,
-    BS: BlobService + Clone,
-    DS: AsRef<dyn DirectoryService>,
-{
-    let entry_stream = walk_path_for_ingestion(blob_service, path.as_ref());
-    ingest_entries(directory_service, entry_stream).await
-}
-
-/// Ingests elements from the given stream of [IngestionEntry] into a the passed [DirectoryService].
+/// Ingests [IngestionEntry] from the given stream into a the passed [DirectoryService].
+/// On success, returns the root [Node].
 ///
 /// The stream must have the following invariants:
 /// - All children entries must come before their parents.
@@ -336,6 +157,31 @@ where
     Ok(root_node)
 }
 
+/// Uploads the file at the provided [Path] the the [BlobService].
+#[instrument(skip(blob_service), fields(path), err)]
+async fn upload_blob_at_path<BS>(blob_service: BS, path: PathBuf) -> Result<B3Digest, Error>
+where
+    BS: BlobService,
+{
+    let mut file = match tokio::fs::File::open(&path).await {
+        Ok(file) => file,
+        Err(e) => return Err(Error::UnableToRead(path, e)),
+    };
+
+    let mut writer = blob_service.open_write().await;
+
+    if let Err(e) = tokio::io::copy(&mut file, &mut writer).await {
+        return Err(Error::UnableToRead(path, e));
+    };
+
+    let digest = writer
+        .close()
+        .await
+        .map_err(|e| Error::UnableToRead(path, e))?;
+
+    Ok(digest)
+}
+
 type BlobFut<'a> = Pin<Box<dyn Future<Output = Result<B3Digest, Error>> + Send + 'a>>;
 
 pub enum IngestionEntry<'a> {
diff --git a/tvix/castore/src/tests/import.rs b/tvix/castore/src/tests/import.rs
index b44b71cd78..8b3bd5ce0f 100644
--- a/tvix/castore/src/tests/import.rs
+++ b/tvix/castore/src/tests/import.rs
@@ -1,7 +1,7 @@
 use crate::blobservice::{self, BlobService};
 use crate::directoryservice;
 use crate::fixtures::*;
-use crate::import::ingest_path;
+use crate::import::fs::ingest_path;
 use crate::proto;
 
 use std::sync::Arc;
diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs
index 46575743c4..f0f2f5cf91 100644
--- a/tvix/glue/src/tvix_store_io.rs
+++ b/tvix/glue/src/tvix_store_io.rs
@@ -18,7 +18,7 @@ use std::{
 use tokio_util::io::SyncIoBridge;
 use tracing::{error, instrument, warn, Level};
 use tvix_build::buildservice::BuildService;
-use tvix_castore::import::dir_entry_iter_to_ingestion_stream;
+use tvix_castore::import::fs::dir_entry_iter_to_ingestion_stream;
 use tvix_eval::{ErrorKind, EvalIO, FileType, StdIO};
 use tvix_store::utils::AsyncIoBridge;
 use walkdir::DirEntry;
diff --git a/tvix/store/src/bin/tvix-store.rs b/tvix/store/src/bin/tvix-store.rs
index 1f172d65c6..7e362576a1 100644
--- a/tvix/store/src/bin/tvix-store.rs
+++ b/tvix/store/src/bin/tvix-store.rs
@@ -17,7 +17,7 @@ use tracing::Level;
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::Layer;
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
-use tvix_castore::import::ingest_path;
+use tvix_castore::import::fs::ingest_path;
 use tvix_store::proto::NarInfo;
 use tvix_store::proto::PathInfo;
 
diff --git a/tvix/store/src/import.rs b/tvix/store/src/import.rs
index 5cff29a9e5..7b6aeb824e 100644
--- a/tvix/store/src/import.rs
+++ b/tvix/store/src/import.rs
@@ -1,7 +1,8 @@
 use std::path::Path;
 use tracing::{debug, instrument};
 use tvix_castore::{
-    blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest,
+    blobservice::BlobService, directoryservice::DirectoryService, import::fs::ingest_path,
+    proto::node::Node, B3Digest,
 };
 
 use nix_compat::{
@@ -116,8 +117,7 @@ where
     DS: AsRef<dyn DirectoryService>,
     PS: AsRef<dyn PathInfoService>,
 {
-    let root_node =
-        tvix_castore::import::ingest_path(blob_service, directory_service, path.as_ref()).await?;
+    let root_node = ingest_path(blob_service, directory_service, path.as_ref()).await?;
 
     // Ask the PathInfoService for the NAR size and sha256
     let (nar_size, nar_sha256) = path_info_service.as_ref().calculate_nar(&root_node).await?;