refactor(tvix): move castore into tvix-castore crate r/6629

This splits the pure content-addressed layers from tvix-store into a
`castore` crate, and only leaves PathInfo related things, as well as the
CLI entrypoint in the tvix-store crate.

Notable changes:
 - `fixtures` and `utils` had to be moved out of the `test` cfg, so they
   can be imported from tvix-store.
 - Some ad-hoc fixtures in the test were moved to proper fixtures in the
   same step.
 - The protos are now created by a (more static) recipe in the protos/
   directory.

The (now two) golang targets are commented out, as it's not possible to
update them properly in the same CL. This will be done by a followup CL
once this is merged (and whitby deployed)

Bug: https://b.tvl.fyi/issues/301

Change-Id: I8d675d4bf1fb697eb7d479747c1b1e3635718107
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9370
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: flokli <flokli@flokli.de>
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>

1 files changed, 147 insertions, 0 deletions

diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index 947f8ab825..2f01a94732 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -53,6 +53,16 @@ rec {
       # File a bug if you depend on any for non-debug work!
       debug = internal.debugCrate { inherit packageId; };
     };
+    "tvix-castore" = rec {
+      packageId = "tvix-castore";
+      build = internal.buildRustCrateWithFeatures {
+        packageId = "tvix-castore";
+      };
+
+      # Debug support which might change between releases.
+      # File a bug if you depend on any for non-debug work!
+      debug = internal.debugCrate { inherit packageId; };
+    };
     "tvix-cli" = rec {
       packageId = "tvix-cli";
       build = internal.buildRustCrateWithFeatures {
@@ -8115,6 +8125,135 @@ rec {
         ];
 
       };
+      "tvix-castore" = rec {
+        crateName = "tvix-castore";
+        version = "0.1.0";
+        edition = "2021";
+        # We can't filter paths with references in Nix 2.4
+        # See https://github.com/NixOS/nix/issues/5410
+        src =
+          if (lib.versionOlder builtins.nixVersion "2.4pre20211007")
+          then lib.cleanSourceWith { filter = sourceFilter; src = ./castore; }
+          else ./castore;
+        dependencies = [
+          {
+            name = "anyhow";
+            packageId = "anyhow";
+          }
+          {
+            name = "async-stream";
+            packageId = "async-stream";
+          }
+          {
+            name = "blake3";
+            packageId = "blake3";
+            features = [ "rayon" "std" ];
+          }
+          {
+            name = "bytes";
+            packageId = "bytes";
+          }
+          {
+            name = "data-encoding";
+            packageId = "data-encoding";
+          }
+          {
+            name = "futures";
+            packageId = "futures";
+          }
+          {
+            name = "lazy_static";
+            packageId = "lazy_static";
+          }
+          {
+            name = "pin-project-lite";
+            packageId = "pin-project-lite";
+          }
+          {
+            name = "prost";
+            packageId = "prost";
+          }
+          {
+            name = "sled";
+            packageId = "sled";
+            features = [ "compression" ];
+          }
+          {
+            name = "thiserror";
+            packageId = "thiserror";
+          }
+          {
+            name = "tokio";
+            packageId = "tokio";
+            features = [ "fs" "net" "rt-multi-thread" "signal" ];
+          }
+          {
+            name = "tokio-stream";
+            packageId = "tokio-stream";
+            features = [ "fs" "net" ];
+          }
+          {
+            name = "tokio-util";
+            packageId = "tokio-util";
+            features = [ "io" "io-util" ];
+          }
+          {
+            name = "tonic";
+            packageId = "tonic";
+          }
+          {
+            name = "tonic-reflection";
+            packageId = "tonic-reflection";
+            optional = true;
+          }
+          {
+            name = "tower";
+            packageId = "tower";
+          }
+          {
+            name = "tracing";
+            packageId = "tracing";
+          }
+          {
+            name = "url";
+            packageId = "url";
+          }
+          {
+            name = "walkdir";
+            packageId = "walkdir";
+          }
+        ];
+        buildDependencies = [
+          {
+            name = "prost-build";
+            packageId = "prost-build";
+          }
+          {
+            name = "tonic-build";
+            packageId = "tonic-build";
+          }
+        ];
+        devDependencies = [
+          {
+            name = "tempfile";
+            packageId = "tempfile";
+          }
+          {
+            name = "test-case";
+            packageId = "test-case";
+          }
+          {
+            name = "tonic-mock";
+            packageId = "tonic-mock";
+          }
+        ];
+        features = {
+          "default" = [ "reflection" ];
+          "reflection" = [ "tonic-reflection" ];
+          "tonic-reflection" = [ "dep:tonic-reflection" ];
+        };
+        resolvedDefaultFeatures = [ "default" "reflection" "tonic-reflection" ];
+      };
       "tvix-cli" = rec {
         crateName = "tvix-cli";
         version = "0.1.0";
@@ -8167,6 +8306,10 @@ rec {
             packageId = "tracing";
           }
           {
+            name = "tvix-castore";
+            packageId = "tvix-castore";
+          }
+          {
             name = "tvix-eval";
             packageId = "tvix-eval";
           }
@@ -8510,6 +8653,10 @@ rec {
             features = [ "json" ];
           }
           {
+            name = "tvix-castore";
+            packageId = "tvix-castore";
+          }
+          {
             name = "url";
             packageId = "url";
           }