Extract NixOS options documentation generation to a function

Motivation is to support other repositories containing nixos
modules that would like to generate options documentation:

- nix-darwin
- private repos
- arion
- ??

3 files changed, 476 insertions, 0 deletions

diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix
new file mode 100644
index 00000000000..c22c7500335
--- /dev/null
+++ b/nixos/lib/make-options-doc/default.nix
@@ -0,0 +1,125 @@
+/* Generate JSON, XML and DocBook documentation for given NixOS options.
+
+   Minimal example:
+
+    { pkgs,  }:
+
+    let
+      eval = import (pkgs.path + "/nixos/lib/eval-config.nix") {
+        baseModules = [
+          ../module.nix
+        ];
+        modules = [];
+      };
+    in pkgs.nixosOptionsDoc {
+      options = eval.options;
+    }
+
+*/
+{ pkgs
+, lib
+, options
+, transformOptions ? lib.id  # function for additional tranformations of the options
+, revision ? "" # Specify revision for the options
+}:
+
+let
+  # Replace functions by the string <function>
+  substFunction = x:
+    if builtins.isAttrs x then lib.mapAttrs (name: substFunction) x
+    else if builtins.isList x then map substFunction x
+    else if lib.isFunction x then "<function>"
+    else x;
+
+  optionsListDesc = lib.flip map optionsListVisible
+   (opt: transformOptions opt
+    // lib.optionalAttrs (opt ? example) { example = substFunction opt.example; }
+    // lib.optionalAttrs (opt ? default) { default = substFunction opt.default; }
+    // lib.optionalAttrs (opt ? type) { type = substFunction opt.type; }
+    // lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != []) { relatedPackages = genRelatedPackages opt.relatedPackages; }
+   );
+
+  # Generate DocBook documentation for a list of packages. This is
+  # what `relatedPackages` option of `mkOption` from
+  # ../../../lib/options.nix influences.
+  #
+  # Each element of `relatedPackages` can be either
+  # - a string:  that will be interpreted as an attribute name from `pkgs`,
+  # - a list:    that will be interpreted as an attribute path from `pkgs`,
+  # - an attrset: that can specify `name`, `path`, `package`, `comment`
+  #   (either of `name`, `path` is required, the rest are optional).
+  genRelatedPackages = packages:
+    let
+      unpack = p: if lib.isString p then { name = p; }
+                  else if lib.isList p then { path = p; }
+                  else p;
+      describe = args:
+        let
+          title = args.title or null;
+          name = args.name or (lib.concatStringsSep "." args.path);
+          path = args.path or [ args.name ];
+          package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}'") pkgs);
+        in "<listitem>"
+        + "<para><literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name} (${package.meta.name})</literal>"
+        + lib.optionalString (!package.meta.available) " <emphasis>[UNAVAILABLE]</emphasis>"
+        + ": ${package.meta.description or "???"}.</para>"
+        + lib.optionalString (args ? comment) "\n<para>${args.comment}</para>"
+        # Lots of `longDescription's break DocBook, so we just wrap them into <programlisting>
+        + lib.optionalString (package.meta ? longDescription) "\n<programlisting>${package.meta.longDescription}</programlisting>"
+        + "</listitem>";
+    in "<itemizedlist>${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}</itemizedlist>";
+
+  # Custom "less" that pushes up all the things ending in ".enable*"
+  # and ".package*"
+  optionLess = a: b:
+    let
+      ise = lib.hasPrefix "enable";
+      isp = lib.hasPrefix "package";
+      cmp = lib.splitByAndCompare ise lib.compare
+                                 (lib.splitByAndCompare isp lib.compare lib.compare);
+    in lib.compareLists cmp a.loc b.loc < 0;
+
+  # Remove invisible and internal options.
+  optionsListVisible = lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList options);
+
+  # Customly sort option list for the man page.
+  optionsList = lib.sort optionLess optionsListDesc;
+
+  # Convert the list of options into an XML file.
+  optionsXML = builtins.toFile "options.xml" (builtins.toXML optionsList);
+
+in rec {
+  # The NixOS options in JSON format.
+  optionsJSON = pkgs.runCommand "options.json"
+    { meta.description = "List of NixOS options in JSON format";
+    }
+    ''
+      # Export list of options in different format.
+      dst=$out/share/doc/nixos
+      mkdir -p $dst
+
+      cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON
+        (builtins.listToAttrs (map (o: { name = o.name; value = removeAttrs o ["name" "visible" "internal"]; }) optionsList))))
+      } $dst/options.json
+
+      mkdir -p $out/nix-support
+      echo "file json $dst/options.json" >> $out/nix-support/hydra-build-products
+    ''; # */
+
+  optionsDocBook = pkgs.runCommand "options-docbook.xml" {} ''
+    optionsXML=${optionsXML}
+    if grep /nixpkgs/nixos/modules $optionsXML; then
+      echo "The manual appears to depend on the location of Nixpkgs, which is bad"
+      echo "since this prevents sharing via the NixOS channel.  This is typically"
+      echo "caused by an option default that refers to a relative path (see above"
+      echo "for hints about the offending path)."
+      exit 1
+    fi
+
+    ${pkgs.libxslt.bin}/bin/xsltproc \
+      --stringparam revision '${revision}' \
+      -o intermediate.xml ${./options-to-docbook.xsl} $optionsXML
+    ${pkgs.libxslt.bin}/bin/xsltproc \
+      -o "$out" ${./postprocess-option-descriptions.xsl} intermediate.xml
+  '';
+}
diff --git a/nixos/lib/make-options-doc/options-to-docbook.xsl b/nixos/lib/make-options-doc/options-to-docbook.xsl
new file mode 100644
index 00000000000..72ac89d4ff6
--- /dev/null
+++ b/nixos/lib/make-options-doc/options-to-docbook.xsl
@@ -0,0 +1,236 @@
+<?xml version="1.0"?>
+
+<xsl:stylesheet version="1.0"
+                xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:str="http://exslt.org/strings"
+                xmlns:xlink="http://www.w3.org/1999/xlink"
+                xmlns:nixos="tag:nixos.org"
+                xmlns="http://docbook.org/ns/docbook"
+                extension-element-prefixes="str"
+                >
+
+  <xsl:output method='xml' encoding="UTF-8" />
+
+  <xsl:param name="revision" />
+  <xsl:param name="program" />
+
+
+  <xsl:template match="/expr/list">
+    <appendix xml:id="appendix-configuration-options">
+      <title>Configuration Options</title>
+      <variablelist xml:id="configuration-variable-list">
+        <xsl:for-each select="attrs">
+          <xsl:variable name="id" select="concat('opt-', str:replace(str:replace(str:replace(str:replace(attr[@name = 'name']/string/@value, '*', '_'), '&lt;', '_'), '>', '_'), '?', '_'))" />
+          <varlistentry>
+            <term xlink:href="#{$id}">
+              <xsl:attribute name="xml:id"><xsl:value-of select="$id"/></xsl:attribute>
+              <option>
+                <xsl:value-of select="attr[@name = 'name']/string/@value" />
+              </option>
+            </term>
+
+            <listitem>
+
+              <nixos:option-description>
+                <para>
+                  <xsl:value-of disable-output-escaping="yes"
+                                select="attr[@name = 'description']/string/@value" />
+                </para>
+              </nixos:option-description>
+
+              <xsl:if test="attr[@name = 'type']">
+                <para>
+                  <emphasis>Type:</emphasis>
+                  <xsl:text> </xsl:text>
+                  <xsl:value-of select="attr[@name = 'type']/string/@value"/>
+                  <xsl:if test="attr[@name = 'readOnly']/bool/@value = 'true'">
+                    <xsl:text> </xsl:text>
+                    <emphasis>(read only)</emphasis>
+                  </xsl:if>
+                </para>
+              </xsl:if>
+
+              <xsl:if test="attr[@name = 'default']">
+                <para>
+                  <emphasis>Default:</emphasis>
+                  <xsl:text> </xsl:text>
+                  <xsl:apply-templates select="attr[@name = 'default']" mode="top" />
+                </para>
+              </xsl:if>
+
+              <xsl:if test="attr[@name = 'example']">
+                <para>
+                  <emphasis>Example:</emphasis>
+                  <xsl:text> </xsl:text>
+                  <xsl:choose>
+                    <xsl:when test="attr[@name = 'example']/attrs[attr[@name = '_type' and string[@value = 'literalExample']]]">
+                      <programlisting><xsl:value-of select="attr[@name = 'example']/attrs/attr[@name = 'text']/string/@value" /></programlisting>
+                    </xsl:when>
+                    <xsl:otherwise>
+                      <xsl:apply-templates select="attr[@name = 'example']" mode="top" />
+                    </xsl:otherwise>
+                  </xsl:choose>
+                </para>
+              </xsl:if>
+
+              <xsl:if test="attr[@name = 'relatedPackages']">
+                <para>
+                  <emphasis>Related packages:</emphasis>
+                  <xsl:text> </xsl:text>
+                  <xsl:value-of disable-output-escaping="yes"
+                                select="attr[@name = 'relatedPackages']/string/@value" />
+                </para>
+              </xsl:if>
+
+              <xsl:if test="count(attr[@name = 'declarations']/list/*) != 0">
+                <para>
+                  <emphasis>Declared by:</emphasis>
+                </para>
+                <xsl:apply-templates select="attr[@name = 'declarations']" />
+              </xsl:if>
+
+              <xsl:if test="count(attr[@name = 'definitions']/list/*) != 0">
+                <para>
+                  <emphasis>Defined by:</emphasis>
+                </para>
+                <xsl:apply-templates select="attr[@name = 'definitions']" />
+              </xsl:if>
+
+            </listitem>
+
+          </varlistentry>
+
+        </xsl:for-each>
+
+      </variablelist>
+    </appendix>
+  </xsl:template>
+
+
+  <xsl:template match="*" mode="top">
+    <xsl:choose>
+      <xsl:when test="string[contains(@value, '&#010;')]">
+<programlisting>
+<xsl:text>''
+</xsl:text><xsl:value-of select='str:replace(string/@value, "${", "&apos;&apos;${")' /><xsl:text>''</xsl:text></programlisting>
+      </xsl:when>
+      <xsl:otherwise>
+        <literal><xsl:apply-templates /></literal>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+
+  <xsl:template match="null">
+    <xsl:text>null</xsl:text>
+  </xsl:template>
+
+
+  <xsl:template match="string">
+    <xsl:choose>
+      <xsl:when test="(contains(@value, '&quot;') or contains(@value, '\')) and not(contains(@value, '&#010;'))">
+        <xsl:text>''</xsl:text><xsl:value-of select='str:replace(@value, "${", "&apos;&apos;${")' /><xsl:text>''</xsl:text>
+      </xsl:when>
+      <xsl:otherwise>
+        <xsl:text>"</xsl:text><xsl:value-of select="str:replace(str:replace(str:replace(str:replace(@value, '\', '\\'), '&quot;', '\&quot;'), '&#010;', '\n'), '$', '\$')" /><xsl:text>"</xsl:text>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+
+  <xsl:template match="int">
+    <xsl:value-of select="@value" />
+  </xsl:template>
+
+
+  <xsl:template match="bool[@value = 'true']">
+    <xsl:text>true</xsl:text>
+  </xsl:template>
+
+
+  <xsl:template match="bool[@value = 'false']">
+    <xsl:text>false</xsl:text>
+  </xsl:template>
+
+
+  <xsl:template match="list">
+    [
+    <xsl:for-each select="*">
+      <xsl:apply-templates select="." />
+      <xsl:text> </xsl:text>
+    </xsl:for-each>
+    ]
+  </xsl:template>
+
+
+  <xsl:template match="attrs[attr[@name = '_type' and string[@value = 'literalExample']]]">
+    <xsl:value-of select="attr[@name = 'text']/string/@value" />
+  </xsl:template>
+
+
+  <xsl:template match="attrs">
+    {
+    <xsl:for-each select="attr">
+      <xsl:value-of select="@name" />
+      <xsl:text> = </xsl:text>
+      <xsl:apply-templates select="*" /><xsl:text>; </xsl:text>
+    </xsl:for-each>
+    }
+  </xsl:template>
+
+
+  <xsl:template match="derivation">
+    <replaceable>(build of <xsl:value-of select="attr[@name = 'name']/string/@value" />)</replaceable>
+  </xsl:template>
+
+  <xsl:template match="attr[@name = 'declarations' or @name = 'definitions']">
+    <simplelist>
+      <xsl:for-each select="list/string">
+        <member><filename>
+          <!-- Hyperlink the filename either to the NixOS Subversion
+          repository (if it’s a module and we have a revision number),
+          or to the local filesystem. -->
+          <xsl:choose>
+            <xsl:when test="not(starts-with(@value, '/'))">
+              <xsl:choose>
+                <xsl:when test="$revision = 'local'">
+                  <xsl:attribute name="xlink:href">https://github.com/NixOS/nixpkgs/blob/master/<xsl:value-of select="@value"/></xsl:attribute>
+                </xsl:when>
+                <xsl:otherwise>
+                  <xsl:attribute name="xlink:href">https://github.com/NixOS/nixpkgs/blob/<xsl:value-of select="$revision"/>/<xsl:value-of select="@value"/></xsl:attribute>
+                </xsl:otherwise>
+              </xsl:choose>
+            </xsl:when>
+            <xsl:when test="$revision != 'local' and $program = 'nixops' and contains(@value, '/nix/')">
+              <xsl:attribute name="xlink:href">https://github.com/NixOS/nixops/blob/<xsl:value-of select="$revision"/>/nix/<xsl:value-of select="substring-after(@value, '/nix/')"/></xsl:attribute>
+            </xsl:when>
+            <xsl:otherwise>
+              <xsl:attribute name="xlink:href">file://<xsl:value-of select="@value"/></xsl:attribute>
+            </xsl:otherwise>
+          </xsl:choose>
+          <!-- Print the filename and make it user-friendly by replacing the
+          /nix/store/<hash> prefix by the default location of nixos
+          sources. -->
+          <xsl:choose>
+            <xsl:when test="not(starts-with(@value, '/'))">
+              &lt;nixpkgs/<xsl:value-of select="@value"/>&gt;
+            </xsl:when>
+            <xsl:when test="contains(@value, 'nixops') and contains(@value, '/nix/')">
+              &lt;nixops/<xsl:value-of select="substring-after(@value, '/nix/')"/>&gt;
+            </xsl:when>
+            <xsl:otherwise>
+              <xsl:value-of select="@value" />
+            </xsl:otherwise>
+          </xsl:choose>
+        </filename></member>
+      </xsl:for-each>
+    </simplelist>
+  </xsl:template>
+
+
+  <xsl:template match="function">
+    <xsl:text>λ</xsl:text>
+  </xsl:template>
+
+
+</xsl:stylesheet>
diff --git a/nixos/lib/make-options-doc/postprocess-option-descriptions.xsl b/nixos/lib/make-options-doc/postprocess-option-descriptions.xsl
new file mode 100644
index 00000000000..1201c7612c2
--- /dev/null
+++ b/nixos/lib/make-options-doc/postprocess-option-descriptions.xsl
@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+
+<xsl:stylesheet version="1.0"
+                xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:str="http://exslt.org/strings"
+                xmlns:exsl="http://exslt.org/common"
+                xmlns:db="http://docbook.org/ns/docbook"
+                xmlns:nixos="tag:nixos.org"
+                extension-element-prefixes="str exsl">
+  <xsl:output method='xml' encoding="UTF-8" />
+
+  <xsl:template match="@*|node()">
+    <xsl:copy>
+      <xsl:apply-templates select="@*|node()" />
+    </xsl:copy>
+  </xsl:template>
+
+  <xsl:template name="break-up-description">
+    <xsl:param name="input" />
+    <xsl:param name="buffer" />
+
+    <!-- Every time we have two newlines following each other, we want to
+         break it into </para><para>. -->
+    <xsl:variable name="parbreak" select="'&#xa;&#xa;'" />
+
+    <!-- Similar to "(head:tail) = input" in Haskell. -->
+    <xsl:variable name="head" select="$input[1]" />
+    <xsl:variable name="tail" select="$input[position() &gt; 1]" />
+
+    <xsl:choose>
+      <xsl:when test="$head/self::text() and contains($head, $parbreak)">
+        <!-- If the haystack provided to str:split() directly starts or
+             ends with $parbreak, it doesn't generate a <token/> for that,
+             so we are doing this here. -->
+        <xsl:variable name="splitted-raw">
+          <xsl:if test="starts-with($head, $parbreak)"><token /></xsl:if>
+          <xsl:for-each select="str:split($head, $parbreak)">
+            <token><xsl:value-of select="node()" /></token>
+          </xsl:for-each>
+          <!-- Something like ends-with($head, $parbreak), but there is
+               no ends-with() in XSLT, so we need to use substring(). -->
+          <xsl:if test="
+            substring($head, string-length($head) -
+                             string-length($parbreak) + 1) = $parbreak
+          "><token /></xsl:if>
+        </xsl:variable>
+        <xsl:variable name="splitted"
+                      select="exsl:node-set($splitted-raw)/token" />
+        <!-- The buffer we had so far didn't contain any text nodes that
+             contain a $parbreak, so we can put the buffer along with the
+             first token of $splitted into a para element. -->
+        <para xmlns="http://docbook.org/ns/docbook">
+          <xsl:apply-templates select="exsl:node-set($buffer)" />
+          <xsl:apply-templates select="$splitted[1]/node()" />
+        </para>
+        <!-- We have already emitted the first splitted result, so the
+             last result is going to be set as the new $buffer later
+             because its contents may not be directly followed up by a
+             $parbreak. -->
+        <xsl:for-each select="$splitted[position() &gt; 1
+                              and position() &lt; last()]">
+          <para xmlns="http://docbook.org/ns/docbook">
+            <xsl:apply-templates select="node()" />
+          </para>
+        </xsl:for-each>
+        <xsl:call-template name="break-up-description">
+          <xsl:with-param name="input" select="$tail" />
+          <xsl:with-param name="buffer" select="$splitted[last()]/node()" />
+        </xsl:call-template>
+      </xsl:when>
+      <!-- Either non-text node or one without $parbreak, which we just
+           want to buffer and continue recursing. -->
+      <xsl:when test="$input">
+        <xsl:call-template name="break-up-description">
+          <xsl:with-param name="input" select="$tail" />
+          <!-- This essentially appends $head to $buffer. -->
+          <xsl:with-param name="buffer">
+            <xsl:if test="$buffer">
+              <xsl:for-each select="exsl:node-set($buffer)">
+                <xsl:apply-templates select="." />
+              </xsl:for-each>
+            </xsl:if>
+            <xsl:apply-templates select="$head" />
+          </xsl:with-param>
+        </xsl:call-template>
+      </xsl:when>
+      <!-- No more $input, just put the remaining $buffer in a para. -->
+      <xsl:otherwise>
+        <para xmlns="http://docbook.org/ns/docbook">
+          <xsl:apply-templates select="exsl:node-set($buffer)" />
+        </para>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template match="nixos:option-description">
+    <xsl:choose>
+      <!--
+        Only process nodes that are comprised of a single <para/> element,
+        because if that's not the case the description already contains
+        </para><para> in between and we need no further processing.
+      -->
+      <xsl:when test="count(db:para) > 1">
+        <xsl:apply-templates select="node()" />
+      </xsl:when>
+      <xsl:otherwise>
+        <xsl:call-template name="break-up-description">
+          <xsl:with-param name="input"
+                          select="exsl:node-set(db:para/node())" />
+        </xsl:call-template>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+</xsl:stylesheet>