zaphyra's git: haumea

loaders: add matching loader (#10)

Co-authored-by: figsoda <figsoda@pm.me>

diff --git a/default.nix b/default.nix
@@ -3,8 +3,16 @@
 let
   load = import ./src/load.nix {
     inherit lib;
-    root.loaders.default = import ./src/loaders {
-      super.defaultWith = import ./src/loaders/__defaultWith.nix {
+    root = {
+      loaders.default = import ./src/loaders {
+        super.defaultWith = import ./src/loaders/__defaultWith.nix {
+          inherit lib;
+        };
+      };
+      matchers.nix = import ./src/matchers/nix.nix {
+        inherit lib;
+      };
+      parsePath = import ./src/__parsePath.nix {
         inherit lib;
       };
     };

diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md
@@ -7,6 +7,7 @@
   - [`load`](api/load.md)
   - [`loadEvalTests`](api/loadEvalTests.md)
   - [`loaders`](api/loaders.md)
+  - [`matchers`](api/matchers.md)
   - [`transformers`](api/transformers.md)
 - [Contributing to Haumea](notes/contributing.md)
 - [See Also](notes/see-also.md)

diff --git a/docs/src/api/load.md b/docs/src/api/load.md
@@ -10,9 +10,11 @@ Arguments:
 
   The directory to load files from.
 
-- (optional) `loader` : `{ self, super, root, ... } -> Path -> a`
+- (optional) `loader` : `({ self, super, root, ... } -> Path -> a) | [ Matcher ]`
 
   Loader for the files, defaults to [`loaders.default`](loaders.html#loadersdefault).
+  It can be either a function that loads of Nix file,
+  or a list of [matchers](matchers.html) that allows you to load any type of file.
 
 - (optional) `inputs` : `{ ... }`
 

@@ -28,7 +30,7 @@ Arguments:
   `cursor` represents the position of the directory being transformed,
   where `[ ]` means root and `[ "foo" "bar" ]` means `root.foo.bar`.
 
-Nix files found in `src` are loaded into an attribute set with the specified `loader`.
+Files found in `src` are loaded into an attribute set with the specified `loader`.
 As an example, the entirety of haumea's API is `load`ed from the
 [src](https://github.com/nix-community/haumea/tree/main/src) directory.
 

diff --git a/docs/src/api/matchers.md b/docs/src/api/matchers.md
@@ -0,0 +1,53 @@
+# Matchers
+
+Type: `{ matches : String -> Bool, loader : { self, super, root, ... } -> Path -> a }`
+
+Matchers allows non-Nix files to be loaded.
+
+This is used for the `loader` option of [`load`],
+which will find the first matcher where `matches` returns `true`,
+and use its `loader` to load the file.
+
+`matches` takes the name of the file with (up to 2) extra preceding `_`s removed.
+For both `bar.nix` and `foo/__bar.nix`, the string `matches` gets will be `bar.nix`.
+
+`loader` works exactly like passing a function to the `loader` option,
+the only difference is that the matcher interface allows loading non-Nix files.
+
+When using matchers, the attribute name will be the file name without its extension,
+which will be `foo` for all of the following files:
+
+- `foo.nix`
+- `bar/_foo.nix`
+- `baz/foo`
+
+Only the last file extension is removed,
+so `far.bar.baz` will have an attribute name of `foo.bar`.
+
+## `matchers.always`
+
+Source: [`src/matchers/always.nix`](https://github.com/nix-community/haumea/blob/main/src/matchers/always.nix)
+
+Type: `({ self, super, root, ... } -> Path -> a }) -> Matcher`
+
+Matches any file name. This can be used as the last matcher as a catch-all.
+
+## `matchers.nix`
+
+Source: [`src/matchers/nix.nix`](https://github.com/nix-community/haumea/blob/main/src/matchers/nix.nix)
+
+Type: `({ self, super, root, ... } -> Path -> a }) -> Matcher`
+
+Matches files that end in `.nix`. This is the default matcher if no matchers are defined.
+
+## `matchers.regex`
+
+Source: [`src/matchers/regex.nix`](https://github.com/nix-community/haumea/blob/main/src/matchers/regex.nix)
+
+Type: `(regex : String) -> ([ String ] -> { self, super, root, ... } -> Path -> a }) -> Matcher`
+
+Matches the file name using the given regex.
+Instead of a regular loader, the function will also take the regex matches
+returned by `builtins.match`, as shown in the type signature (`[ String ]`).
+
+[`load`]: load.html

diff --git a/docs/src/intro/getting-started.md b/docs/src/intro/getting-started.md
@@ -35,7 +35,7 @@ In `flake.nix`, the main thing you want to look at is `lib`:
 ```
 
 `haumea.lib.load` is the main entry point of haumea.
-It loads a directory (`./src`) of Nix files into an attribute set.
+It loads a directory (`./src`) of Nix files[^1] into an attribute set.
 You can see the result of this by running `nix eval .#lib`:
 
 ```nix

@@ -70,4 +70,6 @@ The documentation for [`load`] explains this more thoroughly and talks about som
 [`loadEvalTests`](../api/loadEvalTests.html) instead of [`load`].
 You can run the checks with `nix flake check`.
 
+[^1]: Non-Nix files can also be loaded using [matchers](../api/matchers.html)
+
 [`load`]: ../api/load.html

diff --git a/src/__parsePath.nix b/src/__parsePath.nix
@@ -0,0 +1,57 @@
+{ lib }:
+
+let
+  inherit (builtins)
+    filter
+    genList
+    stringLength
+    substring
+    tail
+    ;
+  inherit (lib)
+    hasPrefix
+    hasSuffix
+    id
+    pipe
+    last
+    removePrefix
+    ;
+in
+
+path: type:
+
+let
+  stripped = removePrefix "_" (removePrefix "_" path);
+in
+
+if stripped == "" then
+  null
+else {
+  inherit stripped;
+
+  name = {
+    directory = stripped;
+
+    regular =
+      let
+        dots = pipe stripped [
+          stringLength
+          (genList id)
+          tail
+          (filter (i: substring i 1 stripped == "."))
+        ];
+      in
+      if hasSuffix "." stripped || dots == [ ] then
+        stripped
+      else
+        substring 0 (last dots) stripped;
+  }.${type};
+
+  visibility =
+    if hasPrefix "__" path then
+      "super"
+    else if hasPrefix "_" path then
+      "root"
+    else
+      "public";
+}

diff --git a/src/load.nix b/src/load.nix
@@ -4,11 +4,11 @@ let
   inherit (builtins)
     all
     attrValues
-    elemAt
+    filter
     foldl'
+    head
     length
     mapAttrs
-    match
     readDir
     ;
   inherit (lib)

@@ -17,7 +17,7 @@ let
     flatten
     flip
     getAttrFromPath
-    hasSuffix
+    isFunction
     nameValuePair
     optionalAttrs
     pipe

@@ -25,19 +25,6 @@ let
     take
     ;
 
-  parsePath = suffix: path:
-    let
-      matches = match ''^(_{0,2})(.+)${suffix}$'' path;
-    in
-    {
-      name = elemAt matches 1;
-      visibility = {
-        "" = "public";
-        "_" = "root";
-        "__" = "super";
-      }.${elemAt matches 0};
-    };
-
   entry = { isDir, path, ... }:
     "${if isDir then "directory" else "file"} '${path}'";
 

@@ -60,42 +47,43 @@ let
     else
       node.content;
 
-  aggregate = { src, loader, inputs, tree }:
+  aggregate = { src, matchers, inputs, tree }:
     let
       aggregateEntry = path: type:
-        if type == "directory" then
-          let
-            parsed = parsePath "" path;
-            inherit (parsed) name visibility;
-          in
-          nameValuePair name {
-            inherit path visibility;
-            isDir = true;
-            children = aggregate {
-              inherit inputs loader;
-              src = src + "/${path}";
-              tree = tree // {
-                pov = tree.pov ++ [ name ];
+        let
+          parsed = root.parsePath path type;
+          inherit (parsed) name visibility stripped;
+          matches = filter (m: m.matches stripped) matchers;
+        in
+        if parsed == null then
+          null
+        else if type == "directory" then
+          nameValuePair name
+            {
+              inherit path visibility;
+              isDir = true;
+              children = aggregate {
+                inherit inputs matchers;
+                src = src + "/${path}";
+                tree = tree // {
+                  pov = tree.pov ++ [ name ];
+                };
               };
-            };
-          }
-        else if type == "regular" && hasSuffix ".nix" path then
-          let
-            parsed = parsePath ''\.nix'' path;
-            inherit (parsed) name visibility;
-            root = view tree;
-          in
-          nameValuePair name {
-            inherit path visibility;
-            isDir = false;
-            content = fix (self:
-              loader
-                (inputs // {
-                  inherit root self;
-                  super = getAttrFromPath tree.pov root;
-                })
-                (src + "/${path}"));
-          }
+            }
+        else if type == "regular" && matches != [ ] then
+          nameValuePair name
+            {
+              inherit path visibility;
+              isDir = false;
+              content = fix (self:
+                (head matches).loader
+                  (inputs // {
+                    inherit self;
+                    super = getAttrFromPath tree.pov (view tree);
+                    root = view tree;
+                  })
+                  (src + "/${path}"));
+            }
         else
           null;
     in

@@ -123,6 +111,7 @@ in
 , inputs ? { }
 , transformer ? [ ]
 }:
+
 let
   transformer' = cursor: flip pipe
     (map (t: t cursor) (flatten transformer));

@@ -139,7 +128,12 @@ view {
   node = fix (node: {
     isDir = true;
     children = aggregate {
-      inherit src loader inputs;
+      inherit src inputs;
+      matchers =
+        if isFunction loader then
+          [ (root.matchers.nix loader) ]
+        else
+          loader;
       tree = {
         pov = [ ];
         transformer = transformer';

diff --git a/src/matchers/always.nix b/src/matchers/always.nix
@@ -0,0 +1,3 @@
+_:
+
+f: { matches = _: true; loader = f; }

diff --git a/src/matchers/nix.nix b/src/matchers/nix.nix
@@ -0,0 +1,15 @@
+{ lib }:
+
+let
+  inherit (builtins)
+    stringLength
+    ;
+  inherit (lib)
+    hasSuffix
+    ;
+in
+
+f: {
+  matches = file: hasSuffix ".nix" file && stringLength file > 4;
+  loader = f;
+}

diff --git a/src/matchers/regex.nix b/src/matchers/regex.nix
@@ -0,0 +1,13 @@
+_:
+
+let
+  inherit (builtins) match;
+in
+
+re: f:
+
+{
+  matches = file: match re file != null;
+  loader = inputs: path:
+    f (match re (baseNameOf path)) inputs path;
+}

diff --git a/tests/matchers/__fixture/bar.yml b/tests/matchers/__fixture/bar.yml
@@ -0,0 +1 @@
+{ "me": "bar" }

diff --git a/tests/matchers/__fixture/baz/.nix b/tests/matchers/__fixture/baz/.nix
@@ -0,0 +1 @@
+baz/.nix

diff --git a/tests/matchers/__fixture/baz/foo.yml. b/tests/matchers/__fixture/baz/foo.yml.
@@ -0,0 +1 @@
+baz/foo.yml.

diff --git a/tests/matchers/__fixture/baz/me.yaml b/tests/matchers/__fixture/baz/me.yaml
@@ -0,0 +1 @@
+"baz.me"

diff --git a/tests/matchers/__fixture/foo.yaml b/tests/matchers/__fixture/foo.yaml
@@ -0,0 +1 @@
+{ "me": "foo" }

diff --git a/tests/matchers/__fixture/rest.nix b/tests/matchers/__fixture/rest.nix
@@ -0,0 +1 @@
+42

diff --git a/tests/matchers/__fixture/yaml b/tests/matchers/__fixture/yaml
@@ -0,0 +1 @@
+yaml

diff --git a/tests/matchers/expected.nix b/tests/matchers/expected.nix
@@ -0,0 +1,11 @@
+{
+  foo."foo.yaml".me = "foo";
+  bar."bar.yml".me = "bar";
+  baz = {
+    ".nix" = ./__fixture/baz/.nix;
+    "foo.yml." = ./__fixture/baz/foo.yml.;
+    me."me.yaml" = "baz.me";
+  };
+  rest = 42;
+  yaml = ./__fixture/yaml;
+}

diff --git a/tests/matchers/expr.nix b/tests/matchers/expr.nix
@@ -0,0 +1,32 @@
+{ haumea, lib }:
+
+let
+  inherit (builtins)
+    elemAt
+    ;
+
+  inherit (lib)
+    importJSON
+    ;
+
+  inherit (haumea) matchers;
+
+  # just loads json, after all
+  fakeLoadYaml = matches: _: path:
+    let
+      basename = elemAt matches 0;
+      ext = elemAt matches 1;
+    in
+    {
+      "${basename}.${ext}" = importJSON path;
+    };
+in
+
+haumea.load {
+  src = ./__fixture;
+  loader = [
+    (matchers.regex ''^(.+)\.(yaml|yml)$'' fakeLoadYaml)
+    (matchers.nix haumea.loaders.default)
+    (matchers.always haumea.loaders.path)
+  ];
+}