feat(nix/buildkite): use keys based on drvPaths where possible r/7939

This will make it easier to emit intra pipeline dependencies based on
the dependencies between derivations contained therein later.

A consequence of this change is that it is no longer possible to have
the same derivation be exposed as multiple steps in the pipeline. I
doubt that having this is very useful, though.

Keys for extraSteps are not changed significantly, but are distinguished
from derivation based steps (with prefix `drv-*`) by their prefix
`extra-step-*`.

Change-Id: I4165900e512b3967fa3ca4cd5bffd44bc15915fc
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11115
Autosubmit: sterni <sternenseemann@systemli.org>
Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com>
Tested-by: BuildkiteCI

2 files changed, 27 insertions, 3 deletions

diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix
index b7715e902f..cb40fb9623 100644
--- a/nix/buildkite/default.nix
+++ b/nix/buildkite/default.nix
@@ -29,6 +29,23 @@ let
   inherit (depot.nix.readTree) mkLabel;
 in
 rec {
+  # Create a unique key for the buildkite pipeline based on the given derivation
+  # or drvPath. A consequence of using such keys is that every derivation may
+  # only be exposed as a single, unique step in the pipeline.
+  keyForDrv = drvOrPath:
+    let
+      drvPath =
+        if lib.isDerivation drvOrPath then drvOrPath.drvPath
+        else if lib.isString drvOrPath then drvOrPath
+        else builtins.throw "keyForDrv: expected string or derivation";
+
+      # Only use the drv hash to prevent escaping problems. Buildkite also has a
+      # limit of 100 characters on keys.
+    in
+    "drv-" + (builtins.substring 0 32
+      (builtins.baseNameOf (unsafeDiscardStringContext drvPath))
+    );
+
   # Given an arbitrary attribute path generate a Nix expression which obtains
   # this from the root of depot (assumed to be ./.). Attributes may be any
   # Nix strings suitable as attribute names, not just Nix literal-safe strings.
@@ -81,7 +98,7 @@ rec {
     in
     {
       label = ":nix: " + label;
-      key = hashString "sha1" label;
+      key = keyForDrv target;
       skip = shouldSkip { inherit label drvPath parentTargetMap; };
       command = mkBuildCommand {
         attrPath = targetAttrPath target;
@@ -394,7 +411,7 @@ rec {
       commandScriptLink = "nix-buildkite-extra-step-command-script";
 
       step = {
-        key = hashString "sha1" "${cfg.label}-${cfg.parentLabel}";
+        key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}";
         label = ":gear: ${cfg.label} (from ${cfg.parentLabel})";
         skip =
           let
diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix
index 33031f94c2..23f56e2f98 100644
--- a/third_party/overlays/tvl.nix
+++ b/third_party/overlays/tvl.nix
@@ -8,7 +8,14 @@ depot.nix.readTree.drvTargets {
     # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443
     withAWS = false;
   });
-  nix = self.nix_2_3;
+  nix = self.nix_2_3 // {
+    # avoid duplicate pipeline step
+    meta = self.nix_2_3.meta or { } // {
+      ci = self.nix_2_3.meta.ci or { } // {
+        skip = true;
+      };
+    };
+  };
   nix_latest = super.nix.override ({
     # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443
     withAWS = false;