nix: Improve clarity of top-level

hacking/nix/default.nix

@@ -28,25 +28,48 @@ in
 
 let
 
-  treeHelpers = import ./tree-helpers.nix { inherit lib; };
+  treeHelpers = rec {
+    mapLeafNodes = f: lib.mapAttrs (k: v:
+      assert lib.isAttrs v;
+      if v ? __leaf
+      then f v
+      else mapLeafNodes f v
+    );
 
-  makeOverridableWith = f: g: x: lib.fix (self: (g self x) // {
-    override = x': makeOverridableWith f g (f x' x);
+    mkLeaf = value: {
+      __leaf = null;
+      inherit value;
+    };
+
+    untree = mapLeafNodes (leafNode: leafNode.value);
+
+    mapLeaves = f: mapLeafNodes (leafNode: mkLeaf (f leafNode.value));
+  };
+
+  makeOverridable' = f: arg: lib.fix (self: (f self arg) // {
+    override = modifyArg: makeOverridable' f (modifyArg arg);
   });
 
-  crossSystems =
+in let
+
+  isCrossSystemActuallyCross =
+    let
+      inherit (nixpkgsFn {}) hostPlatform;
+    in
+      crossSystem: crossSystem != builtins.intersectAttrs crossSystem hostPlatform;
+
+  crossSystemTree =
     with treeHelpers;
     {
       build = mkLeaf null;
       host =
         let
           # Avoid cache misses in cases where buildPlatform == hostPlatform
-          guard = attrs:
-            if attrs == builtins.intersectAttrs attrs this.pkgs.build.hostPlatform
-            then null
-            else attrs
-          ;
-          mkLeafWithGuard = attrs: mkLeaf (guard attrs);
+          guard = crossSystem:
+            if isCrossSystemActuallyCross crossSystem
+            then crossSystem
+            else null;
+          mkLeafWithGuard = crossSystem: mkLeaf (guard crossSystem);
         in {
           aarch64 = {
             none = mkLeafWithGuard {
@@ -146,31 +169,27 @@ let
         };
     };
 
-  baseArgs = selfThis: {
-    nixpkgsArgsFor = crossSystem: {
-      inherit crossSystem;
-      overlays = [
-        (self: super: {
-          thisTopLevel = selfThis;
-          inherit treeHelpers;
-        })
-        (import ./overlay)
-      ];
-    };
-    attrs = {};
-  };
+in let
 
-  mkThis = self: args:
+  f = self: arg:
     let
-      concreteArgs = args self;
+      concreteArg = arg self;
       pkgs = treeHelpers.untree (treeHelpers.mapLeaves (crossSystem:
-        nixpkgsFn (concreteArgs.nixpkgsArgsFor crossSystem)
-      ) crossSystems);
+        nixpkgsFn (concreteArg.nixpkgsArgsForCrossSystem crossSystem)
+      ) crossSystemTree);
     in {
       inherit lib pkgs;
-    } // import ./top-level self // concreteArgs.attrs;
+    } // import ./top-level self // concreteArg.extraAttrs;
 
-  this = makeOverridableWith lib.id mkThis baseArgs;
+  baseArg = self: {
+    nixpkgsArgsForCrossSystem = crossSystem: {
+      inherit crossSystem;
+      overlays = [
+        (import ./overlay)
+      ];
+    };
+    extraAttrs = {};
+  };
 
 in
-  this
+  makeOverridable' f baseArg

hacking/nix/overlay/default.nix

@@ -14,21 +14,13 @@ assert !(super ? scopeName);
 
 {
 
-  "${scopeName}" =
-    let
-      otherSplices = generateSplicesForMkScope scopeName;
-    in
-      lib.makeScopeWithSplicing
-        splicePackages
-        newScope
-        otherSplices
-        (_: {})
-        (_: {})
-        (self: callPackage ../scope {} self // {
-          __dontMashWhenSplicingChildren = true;
-          inherit otherSplices; # for child spliced scopes
-        })
-      ;
+  "${scopeName}" = makeScopeWithSplicing' rec {
+    otherSplices = generateSplicesForMkScope scopeName;
+    f = self: callPackage ../scope {} self // {
+      __dontMashWhenSplicingChildren = true;
+      inherit otherSplices; # for child spliced scopes
+    };
+  };
 
   # Add Python packages needed by the seL4 ecosystem
   pythonPackagesExtensions = super.pythonPackagesExtensions ++ [

hacking/nix/top-level/aggregates.nix

@@ -0,0 +1,156 @@
+#
+# Copyright 2024, Colias Group, LLC
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+self: with self;
+
+let
+
+  aggregate = name: drvs: pkgs.build.writeText "aggregate-${name}" (toString (lib.flatten drvs));
+
+in {
+
+ worldsForEverythingInstances = [
+    pkgs.host.aarch64.none.this.worlds.default
+    pkgs.host.aarch64.none.this.worlds.qemu-arm-virt.microkit
+    pkgs.host.aarch32.none.this.worlds.default
+    pkgs.host.riscv64.default.none.this.worlds.default
+    pkgs.host.riscv64.default.none.this.worlds.qemu-riscv-virt.microkit
+    pkgs.host.riscv32.default.none.this.worlds.default
+    pkgs.host.x86_64.none.this.worlds.default
+  ];
+
+  someConfigurationBuildTests =
+    let
+      worlds = pkgs.host.aarch64.none.this.worlds.qemu-arm-virt.forBuildTests;
+    in
+      lib.forEach (lib.attrValues worlds) (world: [
+        world.sel4-capdl-initializer
+      ] ++ lib.optionals world.worldConfig.isMicrokit [
+        world.instances.microkit.tests.passive-server-with-deferred-action.links
+      ]);
+
+  sel4testInstances = (lib.mapAttrs (k: v: v.this.sel4test.automate) {
+    aarch64 = pkgs.host.aarch64.none;
+    aarch32 = pkgs.host.aarch32.none;
+    riscv64 = pkgs.host.riscv64.default.none;
+    riscv32 = pkgs.host.riscv32.default.none;
+    # TODO figure out why none doesn't build
+    x86_64 = pkgs.host.x86_64.linux;
+    ia32 = pkgs.host.ia32.linux; # no rust support yet
+  });
+
+  sel4testInstancesList = lib.attrValues sel4testInstances;
+
+  prerequisites = aggregate "prerequisites" [
+    pkgs.build.this.qemuForSeL4
+    pkgs.build.this.capdl-tool
+    (builtins.toJSON (pkgs.build.this.vendoredTopLevelLockfile.configFragment))
+
+    (lib.forEach (with pkgs.host; [ aarch64 aarch32 ]) (arch:
+      arch.none.this.platUtils.rpi4.defaultBootLinks
+    ))
+  ];
+
+  incremental = aggregate "incremental" [
+    (lib.forEach worldsForEverythingInstances (world:
+      map (instance: instance.links) world.instances.all
+    ))
+
+    pkgs.build.this.someUnitTests
+
+    someConfigurationBuildTests
+
+    sel4testInstancesList
+
+    (lib.optionals pkgs.build.hostPlatform.isx86_64 [
+      pkgs.build.this.kani
+      pkgs.build.this.verus
+    ])
+
+    pkgs.host.aarch32.none.this.worlds.default.seL4
+    pkgs.host.ia32.none.this.worlds.default.seL4
+
+    pkgs.host.riscv64.imac.none.stdenv
+    pkgs.host.riscv64.gc.none.stdenv
+    pkgs.host.riscv32.imac.none.stdenv
+    pkgs.host.riscv32.imafc.none.stdenv
+
+    example
+    example-rpi4-b-4gb
+  ];
+
+  nonIncremental = aggregate "non-incremental" [
+    html
+  ];
+
+  excess = aggregate "excess" [
+  ];
+
+  everythingExceptNonIncremental = aggregate "everything-except-non-incremental" [
+    prerequisites
+    incremental
+  ];
+
+  everything = aggregate "everything" [
+    everythingExceptNonIncremental
+    nonIncremental
+  ];
+
+  everythingWithExcess = aggregate "everything-with-excess" [
+    everything
+    excess
+  ];
+
+  fastTests =
+    lib.forEach worldsForEverythingInstances
+      (world: world.instances.allAutomationScripts);
+
+  slowTests = sel4testInstancesList;
+
+  runFastTests = mkRunTests (lib.flatten [
+    fastTests
+  ]);
+
+  witnessFastTests = mkWitnessTests (lib.flatten [
+    fastTests
+  ]);
+
+  runTests = mkRunTests (lib.flatten [
+    fastTests
+    slowTests
+  ]);
+
+  witnessTests = mkWitnessTests (lib.flatten [
+    fastTests
+    slowTests
+  ]);
+
+  mkRunTests = scripts:
+    with pkgs.build;
+    writeScript "run-tests" ''
+      #!${runtimeShell}
+      set -eu
+
+      ${lib.concatStrings (lib.forEach scripts (script: ''
+        echo "<<< running test: ${script.testMeta.name or "unnamed"} >>>"
+        ${script}
+      ''))}
+
+      echo
+      echo '# All tests passed.'
+      echo
+    '';
+
+  mkWitnessTest = script:
+    pkgs.build.runCommand "witness-${script.testMeta.name or "unnamed"}" {} ''
+      ${script}
+      touch $out
+    '';
+
+  mkWitnessTests = scripts:
+    pkgs.build.writeText "witness-tests" (lib.concatStrings (map mkWitnessTest scripts));
+
+}