summary refs log tree commit diff
path: root/lib/sources.nix
diff options
context:
space:
mode:
Diffstat (limited to 'lib/sources.nix')
-rw-r--r--lib/sources.nix284
1 files changed, 284 insertions, 0 deletions
diff --git a/lib/sources.nix b/lib/sources.nix
new file mode 100644
index 00000000000..343449d9a60
--- /dev/null
+++ b/lib/sources.nix
@@ -0,0 +1,284 @@
+# Functions for copying sources to the Nix store.
+{ lib }:
+
+# Tested in lib/tests/sources.sh
+let
+  inherit (builtins)
+    hasContext
+    match
+    readDir
+    split
+    storeDir
+    tryEval
+    ;
+  inherit (lib)
+    boolToString
+    filter
+    getAttr
+    isString
+    pathExists
+    readFile
+    ;
+
+  /*
+    Returns the type of a path: regular (for file), symlink, or directory.
+  */
+  pathType = path: getAttr (baseNameOf path) (readDir (dirOf path));
+
+  /*
+    Returns true if the path exists and is a directory, false otherwise.
+  */
+  pathIsDirectory = path: if pathExists path then (pathType path) == "directory" else false;
+
+  /*
+    Returns true if the path exists and is a regular file, false otherwise.
+  */
+  pathIsRegularFile = path: if pathExists path then (pathType path) == "regular" else false;
+
+  /*
+    A basic filter for `cleanSourceWith` that removes
+    directories of version control system, backup files (*~)
+    and some generated files.
+  */
+  cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! (
+    # Filter out version control software files/directories
+    (baseName == ".git" || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")) ||
+    # Filter out editor backup / swap files.
+    lib.hasSuffix "~" baseName ||
+    match "^\\.sw[a-z]$" baseName != null ||
+    match "^\\..*\\.sw[a-z]$" baseName != null ||
+
+    # Filter out generates files.
+    lib.hasSuffix ".o" baseName ||
+    lib.hasSuffix ".so" baseName ||
+    # Filter out nix-build result symlinks
+    (type == "symlink" && lib.hasPrefix "result" baseName) ||
+    # Filter out sockets and other types of files we can't have in the store.
+    (type == "unknown")
+  );
+
+  /*
+    Filters a source tree removing version control files and directories using cleanSourceFilter.
+
+    Example:
+             cleanSource ./.
+  */
+  cleanSource = src: cleanSourceWith { filter = cleanSourceFilter; inherit src; };
+
+  /*
+    Like `builtins.filterSource`, except it will compose with itself,
+    allowing you to chain multiple calls together without any
+    intermediate copies being put in the nix store.
+
+    Example:
+        lib.cleanSourceWith {
+          filter = f;
+          src = lib.cleanSourceWith {
+            filter = g;
+            src = ./.;
+          };
+        }
+        # Succeeds!
+
+        builtins.filterSource f (builtins.filterSource g ./.)
+        # Fails!
+
+  */
+  cleanSourceWith =
+    {
+      # A path or cleanSourceWith result to filter and/or rename.
+      src,
+      # Optional with default value: constant true (include everything)
+      # The function will be combined with the && operator such
+      # that src.filter is called lazily.
+      # For implementing a filter, see
+      # https://nixos.org/nix/manual/#builtin-filterSource
+      # Type: A function (path -> type -> bool)
+      filter ? _path: _type: true,
+      # Optional name to use as part of the store path.
+      # This defaults to `src.name` or otherwise `"source"`.
+      name ? null
+    }:
+    let
+      orig = toSourceAttributes src;
+    in fromSourceAttributes {
+      inherit (orig) origSrc;
+      filter = path: type: filter path type && orig.filter path type;
+      name = if name != null then name else orig.name;
+    };
+
+  /*
+    Add logging to a source, for troubleshooting the filtering behavior.
+    Type:
+      sources.trace :: sourceLike -> Source
+  */
+  trace =
+    # Source to debug. The returned source will behave like this source, but also log its filter invocations.
+    src:
+    let
+      attrs = toSourceAttributes src;
+    in
+      fromSourceAttributes (
+        attrs // {
+          filter = path: type:
+            let
+              r = attrs.filter path type;
+            in
+              builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
+        }
+      ) // {
+        satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
+      };
+
+  /*
+    Filter sources by a list of regular expressions.
+
+    Example: src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]
+  */
+  sourceByRegex = src: regexes:
+    let
+      isFiltered = src ? _isLibCleanSourceWith;
+      origSrc = if isFiltered then src.origSrc else src;
+    in lib.cleanSourceWith {
+      filter = (path: type:
+        let relPath = lib.removePrefix (toString origSrc + "/") (toString path);
+        in lib.any (re: match re relPath != null) regexes);
+      inherit src;
+    };
+
+  /*
+    Get all files ending with the specified suffices from the given
+    source directory or its descendants, omitting files that do not match
+    any suffix. The result of the example below will include files like
+    `./dir/module.c` and `./dir/subdir/doc.xml` if present.
+
+    Type: sourceLike -> [String] -> Source
+
+    Example:
+      sourceFilesBySuffices ./. [ ".xml" ".c" ]
+  */
+  sourceFilesBySuffices =
+    # Path or source containing the files to be returned
+    src:
+    # A list of file suffix strings
+    exts:
+    let filter = name: type:
+      let base = baseNameOf (toString name);
+      in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
+    in cleanSourceWith { inherit filter src; };
+
+  pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success;
+
+  /*
+    Get the commit id of a git repo.
+
+    Example: commitIdFromGitRepo <nixpkgs/.git>
+  */
+  commitIdFromGitRepo =
+    let readCommitFromFile = file: path:
+        let fileName       = toString path + "/" + file;
+            packedRefsName = toString path + "/packed-refs";
+            absolutePath   = base: path:
+              if lib.hasPrefix "/" path
+              then path
+              else toString (/. + "${base}/${path}");
+        in if pathIsRegularFile path
+           # Resolve git worktrees. See gitrepository-layout(5)
+           then
+             let m   = match "^gitdir: (.*)$" (lib.fileContents path);
+             in if m == null
+                then throw ("File contains no gitdir reference: " + path)
+                else
+                  let gitDir      = absolutePath (dirOf path) (lib.head m);
+                      commonDir'' = if pathIsRegularFile "${gitDir}/commondir"
+                                    then lib.fileContents "${gitDir}/commondir"
+                                    else gitDir;
+                      commonDir'  = lib.removeSuffix "/" commonDir'';
+                      commonDir   = absolutePath gitDir commonDir';
+                      refFile     = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
+                  in readCommitFromFile refFile commonDir
+
+           else if pathIsRegularFile fileName
+           # Sometimes git stores the commitId directly in the file but
+           # sometimes it stores something like: «ref: refs/heads/branch-name»
+           then
+             let fileContent = lib.fileContents fileName;
+                 matchRef    = match "^ref: (.*)$" fileContent;
+             in if  matchRef == null
+                then fileContent
+                else readCommitFromFile (lib.head matchRef) path
+
+           else if pathIsRegularFile packedRefsName
+           # Sometimes, the file isn't there at all and has been packed away in the
+           # packed-refs file, so we have to grep through it:
+           then
+             let fileContent = readFile packedRefsName;
+                 matchRef = match "([a-z0-9]+) ${file}";
+                 isRef = s: isString s && (matchRef s) != null;
+                 # there is a bug in libstdc++ leading to stackoverflow for long strings:
+                 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
+                 refs = filter isRef (split "\n" fileContent);
+             in if refs == []
+                then throw ("Could not find " + file + " in " + packedRefsName)
+                else lib.head (matchRef (lib.head refs))
+
+           else throw ("Not a .git directory: " + path);
+    in readCommitFromFile "HEAD";
+
+  pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
+
+  canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
+
+  # -------------------------------------------------------------------------- #
+  # Internal functions
+  #
+
+  # toSourceAttributes : sourceLike -> SourceAttrs
+  #
+  # Convert any source-like object into a simple, singular representation.
+  # We don't expose this representation in order to avoid having a fifth path-
+  # like class of objects in the wild.
+  # (Existing ones being: paths, strings, sources and x//{outPath})
+  # So instead of exposing internals, we build a library of combinator functions.
+  toSourceAttributes = src:
+    let
+      isFiltered = src ? _isLibCleanSourceWith;
+    in
+    {
+      # The original path
+      origSrc = if isFiltered then src.origSrc else src;
+      filter = if isFiltered then src.filter else _: _: true;
+      name = if isFiltered then src.name else "source";
+    };
+
+  # fromSourceAttributes : SourceAttrs -> Source
+  #
+  # Inverse of toSourceAttributes for Source objects.
+  fromSourceAttributes = { origSrc, filter, name }:
+    {
+      _isLibCleanSourceWith = true;
+      inherit origSrc filter name;
+      outPath = builtins.path { inherit filter name; path = origSrc; };
+    };
+
+in {
+  inherit
+    pathType
+    pathIsDirectory
+    pathIsRegularFile
+
+    pathIsGitRepo
+    commitIdFromGitRepo
+
+    cleanSource
+    cleanSourceWith
+    cleanSourceFilter
+    pathHasContext
+    canCleanSource
+
+    sourceByRegex
+    sourceFilesBySuffices
+
+    trace
+    ;
+}