From a629257ec5e28992ad92316362a4f4662a3001a4 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Thu, 24 Dec 2020 02:52:32 +0100
Subject: tree-sitter: rename library.nix to grammar.nix

---
 .../tools/parsing/tree-sitter/default.nix          |  2 +-
 .../tools/parsing/tree-sitter/grammar.nix          | 42 ++++++++++++++++++++++
 .../tools/parsing/tree-sitter/library.nix          | 42 ----------------------
 3 files changed, 43 insertions(+), 43 deletions(-)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammar.nix
 delete mode 100644 pkgs/development/tools/parsing/tree-sitter/library.nix

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 57c12cce93e..2cfa6398113 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -43,7 +43,7 @@ let
 
   builtGrammars = let
     change = name: grammar:
-      callPackage ./library.nix {} {
+      callPackage ./grammar.nix {} {
         language = name;
         inherit version;
         source = fetchGrammar grammar;
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammar.nix b/pkgs/development/tools/parsing/tree-sitter/grammar.nix
new file mode 100644
index 00000000000..f92e0d79426
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammar.nix
@@ -0,0 +1,42 @@
+{ stdenv
+, tree-sitter
+}:
+
+# Build a parser grammar and put the resulting shared object in `$out/parser`
+
+{
+  # language name
+  language
+  # version of tree-sitter
+, version
+  # source for the language grammar
+, source
+}:
+
+stdenv.mkDerivation {
+
+  pname = "${language}-grammar";
+  inherit version;
+
+  src = source;
+
+  buildInputs = [ tree-sitter ];
+
+  dontUnpack = true;
+  configurePhase= ":";
+  buildPhase = ''
+    runHook preBuild
+    scanner_cc="$src/src/scanner.cc"
+    if [ ! -f "$scanner_cc" ]; then
+      scanner_cc=""
+    fi
+    $CC -I$src/src/ -shared -o parser -Os $src/src/parser.c $scanner_cc -lstdc++
+    runHook postBuild
+  '';
+  installPhase = ''
+    runHook preInstall
+    mkdir $out
+    mv parser $out/
+    runHook postInstall
+  '';
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/library.nix b/pkgs/development/tools/parsing/tree-sitter/library.nix
deleted file mode 100644
index f78939b22ff..00000000000
--- a/pkgs/development/tools/parsing/tree-sitter/library.nix
+++ /dev/null
@@ -1,42 +0,0 @@
-{ stdenv
-, tree-sitter
-}:
-
-# Build a parser grammar and put the resulting shared object in `$out/parser`
-
-{
-  # language name
-  language
-  # version of tree-sitter
-, version
-  # source for the language grammar
-, source
-}:
-
-stdenv.mkDerivation {
-
-  pname = "tree-sitter-${language}-library";
-  inherit version;
-
-  src = source;
-
-  buildInputs = [ tree-sitter ];
-
-  dontUnpack = true;
-  configurePhase= ":";
-  buildPhase = ''
-    runHook preBuild
-    scanner_cc="$src/src/scanner.cc"
-    if [ ! -f "$scanner_cc" ]; then
-      scanner_cc=""
-    fi
-    $CC -I$src/src/ -shared -o parser -Os $src/src/parser.c $scanner_cc -lstdc++
-    runHook postBuild
-  '';
-  installPhase = ''
-    runHook preInstall
-    mkdir $out
-    mv parser $out/
-    runHook postInstall
-  '';
-}
-- 
cgit 1.4.1


From c0a4b41afeb066b0d5eb6c0adbf6be6244966a88 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Thu, 24 Dec 2020 03:01:10 +0100
Subject: tree-sitter: improve update script to fetch all available grammars
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The update script would only fetch the few grammars listed in the
tree-sitter repository previously. But the tree-sitter github orga has
a rather large amount of officially supported grammars.

Thus we change the script to query the github APIs for repositories
instead (up to 100 this is supported without paging).

Since the repository list also contains some that are not grammars,
there is a bash script which lists all repos we are aware of and the
ones we want to ignore. It will make sure we don’t forget any
repositories in the future, by comparing to the actual list with jq.
---
 .../tools/parsing/tree-sitter/update.nix           | 94 ++++++++++++++++++++--
 1 file changed, 86 insertions(+), 8 deletions(-)

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 2a3575a44d2..430d2ead026 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -3,11 +3,70 @@
 , src }:
 
 let
-  # print all the grammar names mentioned in the fetch-fixtures script
-  getGrammarNames = writeShellScript "get-grammars.sh" ''
+  # check in the list of grammars, whether we know all of them.
+  checkKnownGrammars = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
-    sed -ne 's/^fetch_grammar \(\S*\).*$/\1/p' \
-      ${src}/script/fetch-fixtures
+    known='
+    [ "tree-sitter-javascript"
+    , "tree-sitter-c"
+    , "tree-sitter-swift"
+    , "tree-sitter-json"
+    , "tree-sitter-cpp"
+    , "tree-sitter-ruby"
+    , "tree-sitter-razor"
+    , "tree-sitter-go"
+    , "tree-sitter-c-sharp"
+    , "tree-sitter-python"
+    , "tree-sitter-typescript"
+    , "tree-sitter-rust"
+    , "tree-sitter-bash"
+    , "tree-sitter-php"
+    , "tree-sitter-java"
+    , "tree-sitter-scala"
+    , "tree-sitter-ocaml"
+    , "tree-sitter-julia"
+    , "tree-sitter-agda"
+    , "tree-sitter-fluent"
+    , "tree-sitter-html"
+    , "tree-sitter-haskell"
+    , "tree-sitter-regex"
+    , "tree-sitter-css"
+    , "tree-sitter-verilog"
+    , "tree-sitter-jsdoc"
+    , "tree-sitter-ql"
+    ]'
+    ignore='
+    [ "tree-sitter"
+    , "tree-sitter-cli"
+    , "tree-sitter-embedded-template"
+    ${/*this is the haskell language bindings, tree-sitter-haskell is the grammar*/""}
+    , "haskell-tree-sitter"
+    ${/*this is the ruby language bindings, tree-sitter-ruby is the grammar*/""}
+    , "ruby-tree-sitter"
+    ${/*this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar*/""}
+    , "rust-tree-sitter"
+    ${/*this is the nodejs language bindings, tree-sitter-javascript is the grammar*/""}
+    , "node-tree-sitter"
+    ${/*this is the python language bindings, tree-sitter-python is the grammar*/""}
+    , "py-tree-sitter"
+    ${/*afl fuzzing for tree sitter*/""}
+    , "afl-tree-sitter"
+    ${/*archived*/""}
+    , "highlight-schema"
+    ${/*website*/""}
+    , "tree-sitter.github.io"
+    ]'
+    res=$(${jq}/bin/jq \
+      --argjson known "$known" \
+      --argjson ignore "$ignore" \
+      '. - ($known + $ignore)' \
+      )
+    if [ ! "$res" == "[]" ]; then
+      echo "These repositories are neither known nor ignored:" 1>&2
+      echo "$res" 1>&2
+      exit 1
+    fi
+    printf '%s' "$known"
   '';
 
   # TODO
@@ -22,7 +81,7 @@ let
     res=$(${curl}/bin/curl \
       --silent \
       "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest")
-    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message')" =~ "rate limit" ]]; then
+    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
@@ -34,6 +93,21 @@ let
     echo "$release"
   '';
 
+  # find the latest repos of a github organization
+  latestGithubRepos = { orga }: writeShellScript "latest-github-repos" ''
+    set -euo pipefail
+    res=$(${curl}/bin/curl \
+      --silent \
+      'https://api.github.com/orgs/${orga}/repos?per_page=100')
+
+    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
+      echo "rate limited" >&2
+    fi
+
+    printf "%s" "$res" | ${jq}/bin/jq 'map(.name)' \
+      || echo "failed $res"
+  '';
+
   # update one tree-sitter grammar repo and print their nix-prefetch-git output
   updateGrammar = { owner }: writeShellScript "update-grammar.sh" ''
     set -euo pipefail
@@ -49,18 +123,22 @@ let
 
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
-    grammarNames=$(${getGrammarNames})
+    echo "fetching list of grammars" 1>&2
+    grammars=$(${latestGithubRepos { orga = "tree-sitter"; }})
+    echo "checking against the list of grammars we know" 1>&2
+    knownGrammars=$(printf '%s' "$grammars" | ${checkKnownGrammars})
+    # change the json list into a item-per-line bash format
+    grammarNames=$(printf '%s' "$knownGrammars" | ${jq}/bin/jq --raw-output '.[]')
     outputDir="${toString ./.}/grammars"
     mkdir -p "$outputDir"
     updateCommand=$(printf \
       '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \
       "$outputDir")
     printf '%s' "$grammarNames" \
-      | ${xe}/bin/xe printf "tree-sitter-%s\n" {} \
       | ${xe}/bin/xe -j2 -s "$updateCommand"
     ( echo "{"
       printf '%s' "$grammarNames" \
-        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./tree-sitter-%s.json));\n" "$1" "$1"'
+        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "$1" "$1"'
       echo "}" ) \
       > "$outputDir/default.nix"
   '';
-- 
cgit 1.4.1


From a32c5d90e7a4c4cc32088b30e591b516fe761952 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Thu, 24 Dec 2020 03:09:25 +0100
Subject: tree-sitter: update grammars

The new update scripts gives us a bunch of new grammars!
---
 .../tools/parsing/tree-sitter/default.nix          | 12 +++++-
 .../tools/parsing/tree-sitter/grammars/default.nix | 44 ++++++++++++++--------
 .../tree-sitter/grammars/tree-sitter-agda.json     | 10 +++++
 .../tree-sitter/grammars/tree-sitter-c-sharp.json  | 10 +++++
 .../tree-sitter/grammars/tree-sitter-cpp.json      |  8 ++--
 .../tree-sitter/grammars/tree-sitter-css.json      | 10 +++++
 .../tree-sitter/grammars/tree-sitter-fluent.json   | 10 +++++
 .../tree-sitter/grammars/tree-sitter-haskell.json  | 10 +++++
 .../grammars/tree-sitter-javascript.json           |  8 ++--
 .../tree-sitter/grammars/tree-sitter-julia.json    | 10 +++++
 .../tree-sitter/grammars/tree-sitter-ocaml.json    | 10 +++++
 .../tree-sitter/grammars/tree-sitter-python.json   |  8 ++--
 .../tree-sitter/grammars/tree-sitter-ql.json       | 10 +++++
 .../tree-sitter/grammars/tree-sitter-razor.json    | 10 +++++
 .../tree-sitter/grammars/tree-sitter-regex.json    | 10 +++++
 .../tree-sitter/grammars/tree-sitter-rust.json     |  8 ++--
 .../tree-sitter/grammars/tree-sitter-scala.json    | 10 +++++
 .../tree-sitter/grammars/tree-sitter-swift.json    | 10 +++++
 .../grammars/tree-sitter-typescript.json           |  8 ++--
 .../tree-sitter/grammars/tree-sitter-verilog.json  | 10 +++++
 .../tools/parsing/tree-sitter/update.nix           |  2 +-
 21 files changed, 189 insertions(+), 39 deletions(-)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 2cfa6398113..38d8beb90db 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -49,8 +49,16 @@ let
         source = fetchGrammar grammar;
       };
   in
-    # typescript doesn't have parser.c in the same place as others
-    lib.mapAttrs change (removeAttrs (import ./grammars) ["typescript"]);
+    lib.mapAttrs change (removeAttrs (import ./grammars) [
+      # TODO these don't have parser.c in the same place as others.
+      # They might require more elaborate builds?
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-typescript"
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-ocaml"
+      # /nix/…/src/parser.c:1:10: fatal error: tree_sitter/parser.h: No such file or directory
+      "tree-sitter-razor"
+    ]);
 
 in rustPlatform.buildRustPackage {
   pname = "tree-sitter";
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
index e9cb0a1db6e..18217f69a5b 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
@@ -1,18 +1,30 @@
 {
-  bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
-  c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
-  cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
-  embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
-  go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
-  html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
-  java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
-  javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
-  jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
-  json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
-  lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
-  php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
-  python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
-  ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
-  rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
-  typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
+  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
+  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
+  tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
+  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
+  tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
+  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
+  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
+  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
+  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
+  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
+  tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
+  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
+  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
+  tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json));
+  tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json));
+  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
+  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
+  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
+  tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json));
+  tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json));
+  tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json));
+  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
+  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
+  tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
 }
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
new file mode 100644
index 00000000000..f16877e9465
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-agda",
+  "rev": "d710ff14d15ddee3764fd73a0837c6c4c8c913e9",
+  "date": "2019-09-20T18:06:06+08:00",
+  "path": "/nix/store/wqz9v9znaiwhhqi19hgig9bn0yvl4i9s-tree-sitter-agda",
+  "sha256": "1wpfj47l97pxk3i9rzdylqipy849r482fnj3lmx8byhalv7z1vm6",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
new file mode 100644
index 00000000000..6498e40497e
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-c-sharp",
+  "rev": "b8bff2a389ecc098dbb7e8abee492816a5eb42db",
+  "date": "2020-11-15T07:54:17+00:00",
+  "path": "/nix/store/h5p71g661hbyzcdcj6xff8y5pcsivpa4-tree-sitter-c-sharp",
+  "sha256": "0x78s2wgd8b6pwjzbmc9fgp0ivdmxv39wikig1m55slai6yq51wh",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
index 16aef8ee33e..a4538b1a932 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-cpp",
-  "rev": "fb8250eef8b4cf1ce104806c50dc206e388b0e72",
-  "date": "2020-11-02T09:46:58-08:00",
-  "path": "/nix/store/qmza43f0fraa111bg50vaxsgnrs5kqs1-tree-sitter-cpp",
-  "sha256": "1sbk5a2p1fh6798naiswsap6fpj7n7bqhwd3xdyi35xf613qpwsi",
+  "rev": "a35a275df92e7583df38f2de2562361f2b69987e",
+  "date": "2020-12-13T11:27:21-08:00",
+  "path": "/nix/store/l0mv4q1xdxz94ym1nl73y52i1yr9zcgi-tree-sitter-cpp",
+  "sha256": "130vizybkm11j3lpzmf183myz0vjxq75mpy6qz48rrkidhnrlryk",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
new file mode 100644
index 00000000000..b8609c0bd17
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-css",
+  "rev": "23f2cb97d47860c517f67f03e1f4b621d5bd2085",
+  "date": "2020-05-14T14:44:30-07:00",
+  "path": "/nix/store/r5pkz9kly0mhgrmqzdzdsr6d1dpqavld-tree-sitter-css",
+  "sha256": "17svpf36p0p7spppzhm3fi833zpdl2l1scg34r6d4vcbv7dknrjy",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
new file mode 100644
index 00000000000..f40b8465b7d
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-fluent",
+  "rev": "858fdd6f1e81992e00d3541bfb31bac9365d7a47",
+  "date": "2018-06-18T13:00:38-07:00",
+  "path": "/nix/store/zbj8abdlrqi9swm8qn8rhpqmjwcz145f-tree-sitter-fluent",
+  "sha256": "0528v9w0cs73p9048xrddb1wpdhr92sn1sw8yyqfrq5sq0danr9k",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
new file mode 100644
index 00000000000..10d2dd64ac2
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-haskell",
+  "rev": "2a0aa1cb5f1b787a4056a29fa0791e87846e33fb",
+  "date": "2018-11-03T09:56:20-07:00",
+  "path": "/nix/store/9xszs4xi51qr72laxr67zxnh8y2br0gy-tree-sitter-haskell",
+  "sha256": "0z0nfip5m0yrjgm49j5nld7jkgyjdmps101xhbng39pwfnh1av83",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
index 68008d393d1..2c563f3fbfa 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-javascript",
-  "rev": "852f11b394804ac2a8986f8bcaafe77753635667",
-  "date": "2020-10-27T13:43:40-04:00",
-  "path": "/nix/store/7a9nd3prxbv2izvilqdxf58his097ak7-tree-sitter-javascript",
-  "sha256": "1cdqx75fm7fvna0iymw7n03a7f6gayfg97qwqz3himi0775fz9ir",
+  "rev": "3f8b62f9befd3cb3b4cb0de22f6595a0aadf76ca",
+  "date": "2020-12-02T10:20:20-08:00",
+  "path": "/nix/store/c17bf7sjq95lank5ygbglv8j48i5z9w3-tree-sitter-javascript",
+  "sha256": "0fjq1jzrzd8c8rfxkh2s25gnqlyc19k3a8i3r1129kakisn1288k",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
new file mode 100644
index 00000000000..2cf5f06dc19
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-julia",
+  "rev": "6a0863f1ce3fcf6f99dc0addb7886dcbd27c5a48",
+  "date": "2020-09-08T19:39:52-07:00",
+  "path": "/nix/store/xn5nii9mi2aw7xdabyxlglha2vk12h1w-tree-sitter-julia",
+  "sha256": "07ds4wzgvnkgkq07izdglkk8lgpqrylvrs96afnwxi56bnzs8sbv",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
new file mode 100644
index 00000000000..32488ebb73e
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ocaml",
+  "rev": "4ec9ee414dadc2b6e7325a9f8124d02f6cd8c250",
+  "date": "2020-09-18T02:13:40+02:00",
+  "path": "/nix/store/dszjdnwnbziqxav2khs85026msm6fasz-tree-sitter-ocaml",
+  "sha256": "0wy85940fhmrnz7c1gk6xkipm8ixzalq5q4i7mcc6wnjiiwq60gx",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
index 402c4c087b1..86551acbee0 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-python",
-  "rev": "d245768132eb6cb74d8a394ca4d29dc57169b096",
-  "date": "2020-10-27T13:20:04-04:00",
-  "path": "/nix/store/afv7ibxpf5c2shafa89cnza88hsfb7kj-tree-sitter-python",
-  "sha256": "075r5i2id2rn76xm3pcrn5cpvj63dlaxcpvphig39a4c9f9hrpdx",
+  "rev": "65b486b42fe0188e4be344092151042bf1bc6d9a",
+  "date": "2020-12-15T09:33:20-08:00",
+  "path": "/nix/store/k9jziqzyxq2bv55pwl03jcmmca83fjyp-tree-sitter-python",
+  "sha256": "1yxqdlmp0jybm7vvza1ni5a320vrviqkd14pnpcrg9ilzq23mlsh",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
new file mode 100644
index 00000000000..27042b6ef95
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ql",
+  "rev": "a0d688d62dcb9cbc7c53f0d98343c458b3776b3d",
+  "date": "2020-09-16T12:56:09-07:00",
+  "path": "/nix/store/dfdaf6wg80dfw5fvdiir7n9nj6j30g3g-tree-sitter-ql",
+  "sha256": "0f6rfhrbvpg8czfa7mld45by3rp628bs6fyl47a8mn18w6x0n5g2",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
new file mode 100644
index 00000000000..6905f4145aa
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-razor",
+  "rev": "60edbd8e798e416f5226a746396efa6a8614fa9b",
+  "date": "2016-07-08T15:17:50-07:00",
+  "path": "/nix/store/2vkxw48wc0xf07awi65r4rg2m1lpqq6c-tree-sitter-razor",
+  "sha256": "1ldsn9nxldxkxdy7irywx88d6a56q05f6907lypzass07piyqp2i",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
new file mode 100644
index 00000000000..d04c1bdab86
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-regex",
+  "rev": "be2e415b5716615530234d179dc27c32b7a1d86b",
+  "date": "2020-05-14T15:05:54-07:00",
+  "path": "/nix/store/acbcq9lgh5pf6hq3329fa253yfvaj6ji-tree-sitter-regex",
+  "sha256": "0qxbafbwfj64bwhj3455jgkidpdzqwrlsqm3rckq2pi75flnkv42",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
index 4e0e2e2f7f4..38c9e44f2f6 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-rust",
-  "rev": "cf47ff80cbd8bdc6dd649a005c5f5d73f9c347e2",
-  "date": "2020-11-04T06:23:40-08:00",
-  "path": "/nix/store/8cvjxw8cz5jrkqwvgq88r7rfm269xwk1-tree-sitter-rust",
-  "sha256": "0yzcag0yy3ncxwpnb3fmsw8bxaidp7z8kbl8wwaxaj2vvxxf6sam",
+  "rev": "8746bd4b584b8063ee8e445bf31015e887417d33",
+  "date": "2020-11-23T17:16:42-08:00",
+  "path": "/nix/store/1zzxvza23wsdyazw47lhjvrs0za6wjpn-tree-sitter-rust",
+  "sha256": "0bhxfyq8ycnp90pqvr6cf3gpq1vax8a34kaq85dmbrc6ar8a7ap6",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
new file mode 100644
index 00000000000..8d9959cf4cf
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-scala",
+  "rev": "211bb726bb5857f872247b600c7c1808e641a8d4",
+  "date": "2020-07-13T13:31:00-07:00",
+  "path": "/nix/store/6q66gzabxazr2581dgp4pp5wwyk2p6mw-tree-sitter-scala",
+  "sha256": "096ps5za8gxmq61gdd3xdk8cif07vb4v8asls2kdwm6jazm82777",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
new file mode 100644
index 00000000000..8f73380e379
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-swift",
+  "rev": "a22fa5e19bae50098e2252ea96cba3aba43f4c58",
+  "date": "2019-10-24T19:04:02-06:00",
+  "path": "/nix/store/pk5xk8yp6vanbar75bhfrs104w0k1ph0-tree-sitter-swift",
+  "sha256": "14b40lmwrnyvdz2wiv684kfh4fvqfhbj1dgrx81ppmy7hsz7jcq7",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
index d31f72754bf..a875232eab6 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-typescript",
-  "rev": "73afadbd117a8e8551758af9c3a522ef46452119",
-  "date": "2020-10-20T12:15:52-07:00",
-  "path": "/nix/store/wrgp1j5l50cigv5cmlxikw693a55i6g9-tree-sitter-typescript",
-  "sha256": "0dsbmcvjzys9s229drp1l8anram7d6nsx1nixl5m9znw54hr4w9p",
+  "rev": "a3a4becef889692724bb1c2191f8fc4bce6573f9",
+  "date": "2020-12-15T13:34:55-08:00",
+  "path": "/nix/store/dra5wbhgxkkhphm1mhgv87hsfw0g18nw-tree-sitter-typescript",
+  "sha256": "0ir7lmzwnbf1zwbl4bfsib1jilg8pvfc1nabaq2n2rixvghlvf3d",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json
new file mode 100644
index 00000000000..7ab79c6f2d5
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-verilog",
+  "rev": "ad551aae2649da56582bc0557478a7dc979c0be3",
+  "date": "2020-10-13T17:40:47-07:00",
+  "path": "/nix/store/nfaxfqrqkxpwaq8rnk7kcp28nnj8y6m2-tree-sitter-verilog",
+  "sha256": "0cy29i200rnc34d237s19r6a1n5vv4d3wgwpbywxg6ahcankc34m",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 430d2ead026..8368857cb8a 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -34,11 +34,11 @@ let
     , "tree-sitter-verilog"
     , "tree-sitter-jsdoc"
     , "tree-sitter-ql"
+    , "tree-sitter-embedded-template"
     ]'
     ignore='
     [ "tree-sitter"
     , "tree-sitter-cli"
-    , "tree-sitter-embedded-template"
     ${/*this is the haskell language bindings, tree-sitter-haskell is the grammar*/""}
     , "haskell-tree-sitter"
     ${/*this is the ruby language bindings, tree-sitter-ruby is the grammar*/""}
-- 
cgit 1.4.1


From d2988dac8eb1f91a522adc35c8594260e06ea8b4 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Sat, 26 Dec 2020 20:11:43 +0100
Subject: tree-sitter/update: factor repo list into nix

We want the ability to add different orga repos as well, and that is a
lot easier on the nix level.
---
 .../tools/parsing/tree-sitter/default.nix          |   4 +-
 .../tools/parsing/tree-sitter/update.nix           | 118 +++++++++++----------
 2 files changed, 65 insertions(+), 57 deletions(-)

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 38d8beb90db..ee0c870722e 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -1,6 +1,6 @@
 { lib, stdenv
 , fetchgit, fetchFromGitHub, fetchurl
-, writeShellScript, runCommand, which
+, writeShellScript, runCommand, which, formats
 , rustPlatform, jq, nix-prefetch-git, xe, curl, emscripten
 , Security
 , callPackage
@@ -29,7 +29,7 @@ let
   };
 
   update-all-grammars = import ./update.nix {
-    inherit writeShellScript nix-prefetch-git curl jq xe src;
+    inherit writeShellScript nix-prefetch-git curl jq xe src formats;
   };
 
   fetchGrammar = (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; });
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 8368857cb8a..cb08969d296 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -1,72 +1,80 @@
-{ writeShellScript, nix-prefetch-git
+{ writeShellScript, nix-prefetch-git, formats
 , curl, jq, xe
 , src }:
 
 let
+  # Grammars we want to fetch from the tree-sitter github orga
+  knownTreeSitterOrgGrammarRepos = [
+    "tree-sitter-javascript"
+    "tree-sitter-c"
+    "tree-sitter-swift"
+    "tree-sitter-json"
+    "tree-sitter-cpp"
+    "tree-sitter-ruby"
+    "tree-sitter-razor"
+    "tree-sitter-go"
+    "tree-sitter-c-sharp"
+    "tree-sitter-python"
+    "tree-sitter-typescript"
+    "tree-sitter-rust"
+    "tree-sitter-bash"
+    "tree-sitter-php"
+    "tree-sitter-java"
+    "tree-sitter-scala"
+    "tree-sitter-ocaml"
+    "tree-sitter-julia"
+    "tree-sitter-agda"
+    "tree-sitter-fluent"
+    "tree-sitter-html"
+    "tree-sitter-haskell"
+    "tree-sitter-regex"
+    "tree-sitter-css"
+    "tree-sitter-verilog"
+    "tree-sitter-jsdoc"
+    "tree-sitter-ql"
+    "tree-sitter-embedded-template"
+  ];
+
+  # repos of the tree-sitter github orga we want to ignore (not grammars)
+  ignoredTreeSitterOrgRepos = [
+    "tree-sitter"
+    "tree-sitter-cli"
+    # this is the haskell language bindings, tree-sitter-haskell is the grammar
+    "haskell-tree-sitter"
+    # this is the ruby language bindings, tree-sitter-ruby is the grammar
+    "ruby-tree-sitter"
+    # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
+    "rust-tree-sitter"
+    # this is the nodejs language bindings, tree-sitter-javascript is the grammar
+    "node-tree-sitter"
+    # this is the python language bindings, tree-sitter-python is the grammar
+    "py-tree-sitter"
+    # afl fuzzing for tree sitter
+    "afl-tree-sitter"
+    # archived
+    "highlight-schema"
+    # website
+    "tree-sitter.github.io"
+  ];
+
+  jsonFile = name: val: (formats.json {}).generate name val;
+
   # check in the list of grammars, whether we know all of them.
   checkKnownGrammars = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
-    known='
-    [ "tree-sitter-javascript"
-    , "tree-sitter-c"
-    , "tree-sitter-swift"
-    , "tree-sitter-json"
-    , "tree-sitter-cpp"
-    , "tree-sitter-ruby"
-    , "tree-sitter-razor"
-    , "tree-sitter-go"
-    , "tree-sitter-c-sharp"
-    , "tree-sitter-python"
-    , "tree-sitter-typescript"
-    , "tree-sitter-rust"
-    , "tree-sitter-bash"
-    , "tree-sitter-php"
-    , "tree-sitter-java"
-    , "tree-sitter-scala"
-    , "tree-sitter-ocaml"
-    , "tree-sitter-julia"
-    , "tree-sitter-agda"
-    , "tree-sitter-fluent"
-    , "tree-sitter-html"
-    , "tree-sitter-haskell"
-    , "tree-sitter-regex"
-    , "tree-sitter-css"
-    , "tree-sitter-verilog"
-    , "tree-sitter-jsdoc"
-    , "tree-sitter-ql"
-    , "tree-sitter-embedded-template"
-    ]'
-    ignore='
-    [ "tree-sitter"
-    , "tree-sitter-cli"
-    ${/*this is the haskell language bindings, tree-sitter-haskell is the grammar*/""}
-    , "haskell-tree-sitter"
-    ${/*this is the ruby language bindings, tree-sitter-ruby is the grammar*/""}
-    , "ruby-tree-sitter"
-    ${/*this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar*/""}
-    , "rust-tree-sitter"
-    ${/*this is the nodejs language bindings, tree-sitter-javascript is the grammar*/""}
-    , "node-tree-sitter"
-    ${/*this is the python language bindings, tree-sitter-python is the grammar*/""}
-    , "py-tree-sitter"
-    ${/*afl fuzzing for tree sitter*/""}
-    , "afl-tree-sitter"
-    ${/*archived*/""}
-    , "highlight-schema"
-    ${/*website*/""}
-    , "tree-sitter.github.io"
-    ]'
+    known="${jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos}"
+    ignore="${jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos}"
     res=$(${jq}/bin/jq \
-      --argjson known "$known" \
-      --argjson ignore "$ignore" \
-      '. - ($known + $ignore)' \
+      --slurpfile known "$known" \
+      --slurpfile ignore "$ignore" \
+      '. - ($known[0] + $ignore[0])' \
       )
     if [ ! "$res" == "[]" ]; then
       echo "These repositories are neither known nor ignored:" 1>&2
       echo "$res" 1>&2
       exit 1
     fi
-    printf '%s' "$known"
+    cat "$known"
   '';
 
   # TODO
-- 
cgit 1.4.1


From 0b1bfd7d146de19d25ee728552c8b88f74a266ca Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Sat, 26 Dec 2020 20:41:19 +0100
Subject: tree-sitter/update: make the repo check into a boolean test

---
 .../tools/parsing/tree-sitter/update.nix           | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index cb08969d296..57a07ee3929 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -4,7 +4,7 @@
 
 let
   # Grammars we want to fetch from the tree-sitter github orga
-  knownTreeSitterOrgGrammarRepos = [
+  knownTreeSitterOrgGrammarRepos = jsonFile "known-tree-sitter-org-grammar-repos" [
     "tree-sitter-javascript"
     "tree-sitter-c"
     "tree-sitter-swift"
@@ -36,7 +36,7 @@ let
   ];
 
   # repos of the tree-sitter github orga we want to ignore (not grammars)
-  ignoredTreeSitterOrgRepos = [
+  ignoredTreeSitterOrgRepos = jsonFile "ignored-tree-sitter-org-repos" [
     "tree-sitter"
     "tree-sitter-cli"
     # this is the haskell language bindings, tree-sitter-haskell is the grammar
@@ -59,14 +59,12 @@ let
 
   jsonFile = name: val: (formats.json {}).generate name val;
 
-  # check in the list of grammars, whether we know all of them.
-  checkKnownGrammars = writeShellScript "get-grammars.sh" ''
+  # check the tree-sitter orga repos
+  checkTreeSitterRepos = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
-    known="${jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos}"
-    ignore="${jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos}"
     res=$(${jq}/bin/jq \
-      --slurpfile known "$known" \
-      --slurpfile ignore "$ignore" \
+      --slurpfile known "${knownTreeSitterOrgGrammarRepos}" \
+      --slurpfile ignore "${ignoredTreeSitterOrgRepos}" \
       '. - ($known[0] + $ignore[0])' \
       )
     if [ ! "$res" == "[]" ]; then
@@ -74,7 +72,6 @@ let
       echo "$res" 1>&2
       exit 1
     fi
-    cat "$known"
   '';
 
   # TODO
@@ -132,9 +129,10 @@ let
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
     echo "fetching list of grammars" 1>&2
-    grammars=$(${latestGithubRepos { orga = "tree-sitter"; }})
-    echo "checking against the list of grammars we know" 1>&2
-    knownGrammars=$(printf '%s' "$grammars" | ${checkKnownGrammars})
+    treeSitterRepos=$(${latestGithubRepos { orga = "tree-sitter"; }})
+    echo "checking the tree-sitter repo list against the grammars we know" 1>&2
+    printf '%s' "$treeSitterRepos" | ${checkTreeSitterRepos}
+    knownGrammars=$(cat "${knownTreeSitterOrgGrammarRepos}")
     # change the json list into a item-per-line bash format
     grammarNames=$(printf '%s' "$knownGrammars" | ${jq}/bin/jq --raw-output '.[]')
     outputDir="${toString ./.}/grammars"
-- 
cgit 1.4.1


From b4ff78a45b29c22029d32186d4f42f28d9be7919 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Mon, 28 Dec 2020 09:50:57 +0100
Subject: tree-sitter/update: pull repo argument into the nix code

---
 .../tools/parsing/tree-sitter/default.nix          |  2 +-
 .../tools/parsing/tree-sitter/update.nix           | 53 ++++++++++------------
 2 files changed, 26 insertions(+), 29 deletions(-)

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index ee0c870722e..a70757ca15e 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -29,7 +29,7 @@ let
   };
 
   update-all-grammars = import ./update.nix {
-    inherit writeShellScript nix-prefetch-git curl jq xe src formats;
+    inherit writeShellScript nix-prefetch-git curl jq xe src formats lib;
   };
 
   fetchGrammar = (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; });
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 57a07ee3929..c63d363f95a 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -1,10 +1,10 @@
-{ writeShellScript, nix-prefetch-git, formats
+{ writeShellScript, nix-prefetch-git, formats, lib
 , curl, jq, xe
 , src }:
 
 let
   # Grammars we want to fetch from the tree-sitter github orga
-  knownTreeSitterOrgGrammarRepos = jsonFile "known-tree-sitter-org-grammar-repos" [
+  knownTreeSitterOrgGrammarRepos = [
     "tree-sitter-javascript"
     "tree-sitter-c"
     "tree-sitter-swift"
@@ -34,9 +34,10 @@ let
     "tree-sitter-ql"
     "tree-sitter-embedded-template"
   ];
+  knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos;
 
   # repos of the tree-sitter github orga we want to ignore (not grammars)
-  ignoredTreeSitterOrgRepos = jsonFile "ignored-tree-sitter-org-repos" [
+  ignoredTreeSitterOrgRepos = [
     "tree-sitter"
     "tree-sitter-cli"
     # this is the haskell language bindings, tree-sitter-haskell is the grammar
@@ -56,6 +57,7 @@ let
     # website
     "tree-sitter.github.io"
   ];
+  ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
 
   jsonFile = name: val: (formats.json {}).generate name val;
 
@@ -63,8 +65,8 @@ let
   checkTreeSitterRepos = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
     res=$(${jq}/bin/jq \
-      --slurpfile known "${knownTreeSitterOrgGrammarRepos}" \
-      --slurpfile ignore "${ignoredTreeSitterOrgRepos}" \
+      --slurpfile known "${knownTreeSitterOrgGrammarReposJson}" \
+      --slurpfile ignore "${ignoredTreeSitterOrgReposJson}" \
       '. - ($known[0] + $ignore[0])' \
       )
     if [ ! "$res" == "[]" ]; then
@@ -76,23 +78,20 @@ let
 
   # TODO
   urlEscape = x: x;
-  # TODO
-  urlEscapeSh = writeShellScript "escape-url" ''printf '%s' "$1"'';
 
   # generic bash script to find the latest github release for a repo
-  latestGithubRelease = { owner }: writeShellScript "latest-github-release" ''
+  latestGithubRelease = { owner, repo }: writeShellScript "latest-github-release" ''
     set -euo pipefail
-    repo="$1"
     res=$(${curl}/bin/curl \
       --silent \
-      "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest")
+      "https://api.github.com/repos/${urlEscape owner}/${urlEscape repo}/releases/latest")
     if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
     # github sometimes returns an empty list even tough there are releases
     if [ "$release" = "null" ]; then
-      echo "uh-oh, latest for $repo is not there, using HEAD" >&2
+      echo "uh-oh, latest for ${owner + "/" + repo} is not there, using HEAD" >&2
       release="HEAD"
     fi
     echo "$release"
@@ -103,10 +102,10 @@ let
     set -euo pipefail
     res=$(${curl}/bin/curl \
       --silent \
-      'https://api.github.com/orgs/${orga}/repos?per_page=100')
+      'https://api.github.com/orgs/${urlEscape orga}/repos?per_page=100')
 
     if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
-      echo "rate limited" >&2
+      echo "rate limited" >&2   #
     fi
 
     printf "%s" "$res" | ${jq}/bin/jq 'map(.name)' \
@@ -114,37 +113,35 @@ let
   '';
 
   # update one tree-sitter grammar repo and print their nix-prefetch-git output
-  updateGrammar = { owner }: writeShellScript "update-grammar.sh" ''
+  updateGrammar = { owner, repo }: writeShellScript "update-grammar.sh" ''
     set -euo pipefail
-    repo="$1"
-    latest="$(${latestGithubRelease { inherit owner; }} "$repo")"
-    echo "Fetching latest release ($latest) of $repo …" >&2
+    latest="$(${latestGithubRelease { inherit owner repo; }})"
+    echo "Fetching latest release ($latest) of ${repo} …" >&2
     ${nix-prefetch-git}/bin/nix-prefetch-git \
       --quiet \
       --no-deepClone \
-      --url "https://github.com/${urlEscape owner}/$(${urlEscapeSh} "$repo")" \
+      --url "https://github.com/${urlEscape owner}/${urlEscape repo}" \
       --rev "$latest"
     '';
 
+  foreachSh = list: f: lib.concatMapStringsSep "\n" f list;
+
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
     echo "fetching list of grammars" 1>&2
     treeSitterRepos=$(${latestGithubRepos { orga = "tree-sitter"; }})
     echo "checking the tree-sitter repo list against the grammars we know" 1>&2
     printf '%s' "$treeSitterRepos" | ${checkTreeSitterRepos}
-    knownGrammars=$(cat "${knownTreeSitterOrgGrammarRepos}")
-    # change the json list into a item-per-line bash format
-    grammarNames=$(printf '%s' "$knownGrammars" | ${jq}/bin/jq --raw-output '.[]')
     outputDir="${toString ./.}/grammars"
+    echo "writing files to $outputDir" 1>&2
     mkdir -p "$outputDir"
-    updateCommand=$(printf \
-      '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \
-      "$outputDir")
-    printf '%s' "$grammarNames" \
-      | ${xe}/bin/xe -j2 -s "$updateCommand"
+    ${foreachSh knownTreeSitterOrgGrammarRepos
+      (repo: ''${updateGrammar { owner = "tree-sitter"; inherit repo; }} > $outputDir/${repo}.json'')}
     ( echo "{"
-      printf '%s' "$grammarNames" \
-        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "$1" "$1"'
+      ${foreachSh knownTreeSitterOrgGrammarRepos
+        (repo: ''
+           # indentation hack
+             printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${repo}" "${repo}"'')}
       echo "}" ) \
       > "$outputDir/default.nix"
   '';
-- 
cgit 1.4.1


From 5f0ace20af45f8c894380d17fae66bca5bfb2f5b Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 29 Dec 2020 12:57:09 +0100
Subject: tree-sitter/update: allow grammars to be fetched from other orgas

Some of the grammars are not in the official orga, like
tree-sitter-lua, so we make sure the updater knows how to fetch them.
---
 .../tools/parsing/tree-sitter/update.nix           | 65 ++++++++++++++++++----
 1 file changed, 53 insertions(+), 12 deletions(-)

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index c63d363f95a..7b0b16b403f 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -59,6 +59,45 @@ let
   ];
   ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
 
+  # Additional grammars that are not in the official github orga.
+  # If you need a grammar that already exists in the official orga,
+  # make sure to give it a different name.
+  otherGrammars = {
+    "tree-sitter-lua" = {
+      orga = "nvim-treesitter";
+      repo = "tree-sitter-lua";
+    };
+  };
+
+  allGrammars =
+    let
+      treeSitterOrgaGrammars =
+        lib.listToAttrs (map (repo:
+          { name = repo;
+            value = {
+              orga = "tree-sitter";
+              inherit repo;
+            };
+          })
+        knownTreeSitterOrgGrammarRepos);
+
+    in
+      mergeAttrsUnique otherGrammars treeSitterOrgaGrammars;
+
+  # TODO: move to lib
+  mergeAttrsUnique = left: right:
+    let intersect = lib.intersectLists (lib.attrNames left) (lib.attrNames right); in
+    assert
+      lib.assertMsg (intersect == [])
+        (lib.concatStringsSep "\n" [
+          "mergeAttrsUnique: keys in attrset overlapping:"
+          "left: ${lib.generators.toPretty {} (lib.getAttrs intersect left)}"
+          "right: ${lib.generators.toPretty {} (lib.getAttrs intersect right)}"
+        ]);
+    left // right;
+
+
+
   jsonFile = name: val: (formats.json {}).generate name val;
 
   # check the tree-sitter orga repos
@@ -80,18 +119,18 @@ let
   urlEscape = x: x;
 
   # generic bash script to find the latest github release for a repo
-  latestGithubRelease = { owner, repo }: writeShellScript "latest-github-release" ''
+  latestGithubRelease = { orga, repo }: writeShellScript "latest-github-release" ''
     set -euo pipefail
     res=$(${curl}/bin/curl \
       --silent \
-      "https://api.github.com/repos/${urlEscape owner}/${urlEscape repo}/releases/latest")
+      "https://api.github.com/repos/${urlEscape orga}/${urlEscape repo}/releases/latest")
     if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
     # github sometimes returns an empty list even tough there are releases
     if [ "$release" = "null" ]; then
-      echo "uh-oh, latest for ${owner + "/" + repo} is not there, using HEAD" >&2
+      echo "uh-oh, latest for ${orga + "/" + repo} is not there, using HEAD" >&2
       release="HEAD"
     fi
     echo "$release"
@@ -113,18 +152,20 @@ let
   '';
 
   # update one tree-sitter grammar repo and print their nix-prefetch-git output
-  updateGrammar = { owner, repo }: writeShellScript "update-grammar.sh" ''
+  updateGrammar = { orga, repo }: writeShellScript "update-grammar.sh" ''
     set -euo pipefail
-    latest="$(${latestGithubRelease { inherit owner repo; }})"
+    latest="$(${latestGithubRelease { inherit orga repo; }})"
     echo "Fetching latest release ($latest) of ${repo} …" >&2
     ${nix-prefetch-git}/bin/nix-prefetch-git \
       --quiet \
       --no-deepClone \
-      --url "https://github.com/${urlEscape owner}/${urlEscape repo}" \
+      --url "https://github.com/${urlEscape orga}/${urlEscape repo}" \
       --rev "$latest"
     '';
 
-  foreachSh = list: f: lib.concatMapStringsSep "\n" f list;
+  foreachSh = attrs: f:
+    lib.concatMapStringsSep "\n" f
+    (lib.mapAttrsToList (k: v: { name = k; } // v) attrs);
 
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
@@ -135,13 +176,13 @@ let
     outputDir="${toString ./.}/grammars"
     echo "writing files to $outputDir" 1>&2
     mkdir -p "$outputDir"
-    ${foreachSh knownTreeSitterOrgGrammarRepos
-      (repo: ''${updateGrammar { owner = "tree-sitter"; inherit repo; }} > $outputDir/${repo}.json'')}
+    ${foreachSh allGrammars
+      ({name, orga, repo}: ''${updateGrammar { inherit orga repo; }} > $outputDir/${name}.json'')}
     ( echo "{"
-      ${foreachSh knownTreeSitterOrgGrammarRepos
-        (repo: ''
+      ${foreachSh allGrammars
+        ({name, ...}: ''
            # indentation hack
-             printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${repo}" "${repo}"'')}
+             printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${name}" "${name}"'')}
       echo "}" ) \
       > "$outputDir/default.nix"
   '';
-- 
cgit 1.4.1


From 0181e103b37abf085dfc127c14a28b86d1077780 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 29 Dec 2020 13:01:56 +0100
Subject: tree-sitter: update grammars again with new update script

---
 .../tools/parsing/tree-sitter/grammars/default.nix | 43 +++++++++++-----------
 .../tree-sitter/grammars/tree-sitter-lua.json      |  8 ++--
 .../tree-sitter/grammars/tree-sitter-ruby.json     |  8 ++--
 3 files changed, 30 insertions(+), 29 deletions(-)

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
index 18217f69a5b..e1648b84be5 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
@@ -1,30 +1,31 @@
 {
-  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json));
+  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
   tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
-  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
-  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
+  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
   tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
-  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
-  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
+  tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json));
+  tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
+  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
   tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
-  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
-  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
-  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
-  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
-  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
-  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
+  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
+  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
   tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
-  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
-  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
+  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
+  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
   tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json));
-  tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json));
-  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
-  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
-  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
+  tree-sitter-lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
+  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
+  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
+  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
+  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
+  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
   tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json));
-  tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json));
+  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
+  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
+  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
+  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
   tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json));
-  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
-  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
-  tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
 }
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
index 267d25a52a0..804d613357e 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/nvim-treesitter/tree-sitter-lua",
-  "rev": "278b0ea1f1f3d86afc86faaca0cbbf6c01182a54",
-  "date": "2020-09-08T18:29:23+02:00",
-  "path": "/nix/store/xkaqsw030bf6zd6pivrzrg3d50j1ka1k-tree-sitter-lua",
-  "sha256": "0i8jdp0bbl02h58hfhpild4v3rvy8yk5r5l90navvfxw4ad4f6f3",
+  "rev": "97b757ad3546d68dc1131e6ffb9b1d2a750eea29",
+  "date": "2020-11-25T21:23:57+01:00",
+  "path": "/nix/store/irz0dikk9dahkayi41chcznqq1i3wr84-tree-sitter-lua",
+  "sha256": "1nhg139vk3xm6ip4mhr29z0rprfg4q417z2vlvkz3m9wp8gppzls",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
index d5a1767a23c..027478e99f8 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-ruby",
-  "rev": "f86d90cce578e28c824afdcdbdca7d031b88c0b0",
-  "date": "2020-11-02T20:49:49-08:00",
-  "path": "/nix/store/gs0hhb1dlgmv0y60zyyz7ki30yvpddfx-tree-sitter-ruby",
-  "sha256": "1n1irdjfmv27p5ia2vqzv8zfsq19pxi0as0wrkb7z7s1lqr1gwrb",
+  "rev": "bb572f60e9538bd11fbde95a54f97522073f1e06",
+  "date": "2020-12-20T12:25:50-08:00",
+  "path": "/nix/store/hkjgvxh8l4drk1z4zkhmd0wcj876x5y1-tree-sitter-ruby",
+  "sha256": "198sfmrb3mxvpfmlwmwfmb3qs075ih0bjmr0ffrdy4nqm4a00cb8",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
-- 
cgit 1.4.1


From 00359f25e2e528530c70cecf302186e0e49cc1d4 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 29 Dec 2020 13:34:32 +0100
Subject: tree-sitter: test building of grammars

---
 pkgs/development/tools/parsing/tree-sitter/default.nix | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'pkgs/development/tools')

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index a70757ca15e..72150b5ac0b 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -101,6 +101,11 @@ in rustPlatform.buildRustPackage {
     };
     inherit grammars;
     inherit builtGrammars;
+
+    tests = {
+      # make sure all grammars build
+      builtGrammars = lib.recurseIntoAttrs builtGrammars;
+    };
   };
 
   meta = {
-- 
cgit 1.4.1