From 79698c470cf9e043204741b727ce34041fcb1e32 Mon Sep 17 00:00:00 2001 From: Connor Brewster Date: Sat, 20 Apr 2024 16:41:49 -0500 Subject: feat(tvix/castore): upload blobs concurrently when ingesting archives Ingesting tarballs with a lot of small files is very slow because of the round trip time to the `BlobService`. To mitigate this, small blobs can be buffered into memory and uploaded concurrently in the background. Change-Id: I3376d11bb941ae35377a089b96849294c9c139e6 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11497 Reviewed-by: flokli Tested-by: BuildkiteCI Autosubmit: Connor Brewster --- tvix/glue/src/fetchers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tvix/glue/src/fetchers.rs') diff --git a/tvix/glue/src/fetchers.rs b/tvix/glue/src/fetchers.rs index 7981770eb3..7560c447d8 100644 --- a/tvix/glue/src/fetchers.rs +++ b/tvix/glue/src/fetchers.rs @@ -168,7 +168,7 @@ async fn hash( impl Fetcher where - BS: AsRef<(dyn BlobService + 'static)> + Send + Sync, + BS: AsRef<(dyn BlobService + 'static)> + Clone + Send + Sync + 'static, DS: AsRef<(dyn DirectoryService + 'static)>, PS: PathInfoService, { @@ -242,7 +242,7 @@ where // Ingest the archive, get the root node let node = tvix_castore::import::archive::ingest_archive( - &self.blob_service, + self.blob_service.clone(), &self.directory_service, archive, ) -- cgit 1.4.1