From 2798803f76687bdeceb10cdaf31ef3013d30acbc Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 19 Mar 2024 11:46:12 +0200 Subject: refactor(tvix/castore): introduce "cloud" feature flag This controls whether tvix-castore has support for various cloud backends or not. Use this to control the set of feature flags for the object_store backend, and only enable the aws, azure and gcp ones if it's set. In the future this can be used to enable/disable other cloud backends too. Without feature flags, `object_store` already supports the `InMemory` and `LocalFilesystem` backends, and we also want to unconditionally enable the `http` one. Make sure at least the construction of these services is covered in the tests. Similarly, the tvix-store crate, which provides the tvix-store CLI has a `cloud` feature flag too (defaulting to enabled). Change-Id: I9fb9c87b740e7dc83f8ff7a0862905d036d513f2 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11204 Autosubmit: flokli Reviewed-by: raitobezarius Tested-by: BuildkiteCI --- tvix/Cargo.nix | 10 ++++++---- tvix/castore/Cargo.toml | 7 ++++++- tvix/castore/src/blobservice/from_addr.rs | 22 +++++++++++++++++----- tvix/store/Cargo.toml | 3 ++- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 31d6faec25..d3272843c2 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -12117,7 +12117,7 @@ rec { { name = "object_store"; packageId = "object_store"; - features = [ "aws" "azure" "gcp" "http" ]; + features = [ "http" ]; } { name = "parking_lot"; @@ -12243,12 +12243,13 @@ rec { } ]; features = { + "cloud" = [ "object_store/aws" "object_store/azure" "object_store/gcp" ]; "fs" = [ "dep:libc" "dep:fuse-backend-rs" ]; "fuse" = [ "fs" ]; "tonic-reflection" = [ "dep:tonic-reflection" ]; "virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ]; }; - resolvedDefaultFeatures = [ "default" "fs" "fuse" "tonic-reflection" "virtiofs" ]; + resolvedDefaultFeatures = [ "cloud" "default" "fs" "fuse" "tonic-reflection" "virtiofs" ]; }; "tvix-cli" = rec { crateName = "tvix-cli"; @@ -12927,13 +12928,14 @@ rec { } ]; features = { - "default" = [ "fuse" "otlp" "tonic-reflection" ]; + "cloud" = [ "tvix-castore/cloud" ]; + "default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ]; "fuse" = [ "tvix-castore/fuse" ]; "otlp" = [ "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" ]; "tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ]; "virtiofs" = [ "tvix-castore/virtiofs" ]; }; - resolvedDefaultFeatures = [ "default" "fuse" "otlp" "tonic-reflection" "virtiofs" ]; + resolvedDefaultFeatures = [ "cloud" "default" "fuse" "otlp" "tonic-reflection" "virtiofs" ]; }; "typenum" = rec { crateName = "typenum"; diff --git a/tvix/castore/Cargo.toml b/tvix/castore/Cargo.toml index 69834ed2e3..0a8b9b4157 100644 --- a/tvix/castore/Cargo.toml +++ b/tvix/castore/Cargo.toml @@ -14,7 +14,7 @@ digest = "0.10.7" fastcdc = { version = "3.1.0", features = ["tokio"] } futures = "0.3.30" lazy_static = "1.4.0" -object_store = { version = "0.9.1", features = ["aws", "azure", "gcp", "http"] } +object_store = { version = "0.9.1", features = ["http"] } parking_lot = "0.12.1" pin-project-lite = "0.2.13" prost = "0.12.1" @@ -78,6 +78,11 @@ hex-literal = "0.4.1" [features] default = [] +cloud = [ + "object_store/aws", + "object_store/azure", + "object_store/gcp", +] fs = ["dep:libc", "dep:fuse-backend-rs"] virtiofs = [ "fs", diff --git a/tvix/castore/src/blobservice/from_addr.rs b/tvix/castore/src/blobservice/from_addr.rs index 6925407552..1f37ef61ed 100644 --- a/tvix/castore/src/blobservice/from_addr.rs +++ b/tvix/castore/src/blobservice/from_addr.rs @@ -131,17 +131,29 @@ mod tests { /// Correct scheme to connect to localhost over http, without specifying a port. #[test_case("grpc+https://localhost", true; "grpc valid https host without port")] /// Correct scheme to connect to localhost over http, but with additional path, which is invalid. - #[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")] - /// An example for object store (Memory) + #[test_case("grpc+http://localhost/some-path", false; "grpc invalid has path")] + /// An example for object store (InMemory) #[test_case("objectstore+memory:///", true; "objectstore valid memory url")] - /// An example for object store (File) - #[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] // ?? + /// An example for object store (LocalFileSystem) + #[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] + // An example for object store (HTTP / WebDAV) + #[test_case("objectstore+https://localhost:8080/some-path", true; "objectstore valid http url")] + #[tokio::test] + async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) { + if exp_succeed { + from_addr(uri_str).await.expect("should succeed"); + } else { + assert!(from_addr(uri_str).await.is_err(), "should fail"); + } + } + + #[cfg(feature = "cloud")] /// An example for object store (S3) #[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")] /// An example for object store (GCS) #[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")] #[tokio::test] - async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) { + async fn test_from_addr_tokio_cloud(uri_str: &str, exp_succeed: bool) { if exp_succeed { from_addr(uri_str).await.expect("should succeed"); } else { diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index 366bd39216..2b93cd7a52 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -53,7 +53,8 @@ tempfile = "3.3.0" tokio-retry = "0.3.0" [features] -default = ["fuse", "otlp", "tonic-reflection"] +default = ["cloud", "fuse", "otlp", "tonic-reflection"] +cloud = ["tvix-castore/cloud"] fuse = ["tvix-castore/fuse"] otlp = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk"] tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"] -- cgit 1.4.1